2015-07-12 20:36:46 +02:00
|
|
|
import logging
|
|
|
|
import time
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
import sys
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
|
|
|
import gevent
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
from cStringIO import StringIO
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
2015-07-25 13:38:58 +02:00
|
|
|
from Config import config
|
2015-09-27 02:08:53 +02:00
|
|
|
from util import helper
|
2015-10-11 02:22:53 +02:00
|
|
|
from PeerHashfield import PeerHashfield
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-25 13:38:58 +02:00
|
|
|
if config.use_tempfiles:
|
|
|
|
import tempfile
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# Communicate remote peers
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
class Peer(object):
|
2015-07-25 13:38:58 +02:00
|
|
|
__slots__ = (
|
2016-11-07 22:34:46 +01:00
|
|
|
"ip", "port", "site", "key", "connection", "connection_server", "time_found", "time_response", "time_hashfield", "time_added", "has_hashfield",
|
2017-02-27 00:06:13 +01:00
|
|
|
"time_my_hashfield_sent", "last_ping", "reputation", "last_content_json_update", "hashfield", "connection_error", "hash_failed", "download_bytes", "download_time"
|
2015-07-25 13:38:58 +02:00
|
|
|
)
|
2015-06-18 00:01:56 +02:00
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
def __init__(self, ip, port, site=None, connection_server=None):
|
2015-06-18 00:01:56 +02:00
|
|
|
self.ip = ip
|
|
|
|
self.port = port
|
|
|
|
self.site = site
|
|
|
|
self.key = "%s:%s" % (ip, port)
|
|
|
|
|
|
|
|
self.connection = None
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
self.connection_server = connection_server
|
2016-11-07 22:34:46 +01:00
|
|
|
self.has_hashfield = False # Lazy hashfield object not created yet
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
self.time_hashfield = None # Last time peer's hashfiled downloaded
|
|
|
|
self.time_my_hashfield_sent = None # Last time my hashfield sent to peer
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_found = time.time() # Time of last found in the torrent tracker
|
|
|
|
self.time_response = None # Time of last successful response from peer
|
|
|
|
self.time_added = time.time()
|
2015-06-18 00:03:11 +02:00
|
|
|
self.last_ping = None # Last response time for ping
|
2017-02-27 00:06:13 +01:00
|
|
|
self.reputation = 0 # More likely to connect if larger
|
2016-03-30 23:05:43 +02:00
|
|
|
self.last_content_json_update = 0.0 # Modify date of last received content.json
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2015-06-18 00:03:11 +02:00
|
|
|
self.connection_error = 0 # Series of connection error
|
|
|
|
self.hash_failed = 0 # Number of bad files from peer
|
|
|
|
self.download_bytes = 0 # Bytes downloaded
|
|
|
|
self.download_time = 0 # Time spent to download
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2016-09-07 17:41:33 +02:00
|
|
|
def __getattr__(self, key):
|
|
|
|
if key == "hashfield":
|
2016-11-07 22:34:46 +01:00
|
|
|
self.has_hashfield = True
|
2016-09-07 17:41:33 +02:00
|
|
|
self.hashfield = PeerHashfield()
|
|
|
|
return self.hashfield
|
|
|
|
else:
|
|
|
|
return getattr(self, key)
|
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
def log(self, text):
|
2016-03-06 00:55:50 +01:00
|
|
|
if not config.verbose:
|
|
|
|
return # Only log if we are in debug mode
|
2015-06-18 00:01:56 +02:00
|
|
|
if self.site:
|
|
|
|
self.site.log.debug("%s:%s %s" % (self.ip, self.port, text))
|
|
|
|
else:
|
|
|
|
logging.debug("%s:%s %s" % (self.ip, self.port, text))
|
|
|
|
|
|
|
|
# Connect to host
|
|
|
|
def connect(self, connection=None):
|
|
|
|
if self.connection:
|
|
|
|
self.log("Getting connection (Closing %s)..." % self.connection)
|
2017-02-27 00:02:24 +01:00
|
|
|
self.connection.close("Connection change")
|
2015-06-18 00:01:56 +02:00
|
|
|
else:
|
|
|
|
self.log("Getting connection...")
|
|
|
|
|
2015-06-18 00:03:11 +02:00
|
|
|
if connection: # Connection specified
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection = connection
|
2016-04-25 02:23:06 +02:00
|
|
|
self.connection.sites += 1
|
2015-06-18 00:01:56 +02:00
|
|
|
else: # Try to find from connection pool or create new connection
|
|
|
|
self.connection = None
|
|
|
|
|
|
|
|
try:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if self.connection_server:
|
|
|
|
self.connection = self.connection_server.getConnection(self.ip, self.port, site=self.site)
|
|
|
|
elif self.site:
|
|
|
|
self.connection = self.site.connection_server.getConnection(self.ip, self.port, site=self.site)
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
else:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
self.connection = sys.modules["main"].file_server.getConnection(self.ip, self.port, site=self.site)
|
2016-04-25 02:23:06 +02:00
|
|
|
self.connection.sites += 1
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
except Exception, err:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Getting connection error")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
|
|
|
|
(Debug.formatException(err), self.connection_error, self.hash_failed))
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection = None
|
|
|
|
|
|
|
|
# Check if we have connection to peer
|
|
|
|
def findConnection(self):
|
2015-06-18 00:03:11 +02:00
|
|
|
if self.connection and self.connection.connected: # We have connection to peer
|
2015-06-18 00:01:56 +02:00
|
|
|
return self.connection
|
2015-06-18 00:03:11 +02:00
|
|
|
else: # Try to find from other sites connections
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False, site=self.site)
|
2016-05-05 12:12:18 +02:00
|
|
|
if self.connection:
|
|
|
|
self.connection.sites += 1
|
2015-06-18 00:01:56 +02:00
|
|
|
return self.connection
|
|
|
|
|
|
|
|
def __str__(self):
|
2015-08-06 00:51:25 +02:00
|
|
|
return "Peer:%-12s" % self.ip
|
2015-06-18 00:01:56 +02:00
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return "<%s>" % self.__str__()
|
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
def packMyAddress(self):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if self.ip.endswith(".onion"):
|
|
|
|
return helper.packOnionAddress(self.ip, self.port)
|
|
|
|
else:
|
|
|
|
return helper.packAddress(self.ip, self.port)
|
2015-06-18 00:01:56 +02:00
|
|
|
|
|
|
|
# Found a peer on tracker
|
|
|
|
def found(self):
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_found = time.time()
|
2015-06-18 00:01:56 +02:00
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Send a command to peer and return response value
|
2015-07-25 13:38:58 +02:00
|
|
|
def request(self, cmd, params={}, stream_to=None):
|
2015-06-18 00:01:56 +02:00
|
|
|
if not self.connection or self.connection.closed:
|
|
|
|
self.connect()
|
|
|
|
if not self.connection:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Reconnect error")
|
2015-07-12 20:36:46 +02:00
|
|
|
return None # Connection failed
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2016-03-06 00:55:50 +01:00
|
|
|
self.log("Send request: %s %s" % (params.get("site", ""), cmd))
|
2016-03-03 21:12:16 +01:00
|
|
|
|
2015-11-05 23:19:36 +01:00
|
|
|
for retry in range(1, 4): # Retry 3 times
|
2015-06-18 00:01:56 +02:00
|
|
|
try:
|
2017-06-13 14:12:48 +02:00
|
|
|
if not self.connection:
|
|
|
|
raise Exception("No connection found")
|
2015-10-22 11:42:55 +02:00
|
|
|
res = self.connection.request(cmd, params, stream_to)
|
|
|
|
if not res:
|
2015-06-18 00:01:56 +02:00
|
|
|
raise Exception("Send error")
|
2015-10-22 11:42:55 +02:00
|
|
|
if "error" in res:
|
|
|
|
self.log("%s error: %s" % (cmd, res["error"]))
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Response error")
|
|
|
|
break
|
2015-07-12 20:36:46 +02:00
|
|
|
else: # Successful request, reset connection error num
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection_error = 0
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_response = time.time()
|
|
|
|
return res
|
2015-06-18 00:01:56 +02:00
|
|
|
except Exception, err:
|
2015-06-18 00:03:11 +02:00
|
|
|
if type(err).__name__ == "Notify": # Greenlet killed by worker
|
2015-06-18 00:01:56 +02:00
|
|
|
self.log("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd))
|
|
|
|
break
|
|
|
|
else:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Request error")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log(
|
|
|
|
"%s (connection_error: %s, hash_failed: %s, retry: %s)" %
|
|
|
|
(Debug.formatException(err), self.connection_error, self.hash_failed, retry)
|
|
|
|
)
|
|
|
|
time.sleep(1 * retry)
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connect()
|
|
|
|
return None # Failed after 4 retry
|
|
|
|
|
|
|
|
# Get a file content from peer
|
2017-06-19 16:10:18 +02:00
|
|
|
def getFile(self, site, inner_path, file_size=None):
|
2015-07-25 13:38:58 +02:00
|
|
|
# Use streamFile if client supports it
|
2015-08-06 00:51:25 +02:00
|
|
|
if config.stream_downloads and self.connection and self.connection.handshake and self.connection.handshake["rev"] > 310:
|
2015-07-25 13:38:58 +02:00
|
|
|
return self.streamFile(site, inner_path)
|
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
location = 0
|
2015-07-25 13:38:58 +02:00
|
|
|
if config.use_tempfiles:
|
|
|
|
buff = tempfile.SpooledTemporaryFile(max_size=16 * 1024, mode='w+b')
|
|
|
|
else:
|
|
|
|
buff = StringIO()
|
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
s = time.time()
|
2015-06-18 00:06:41 +02:00
|
|
|
while True: # Read in 512k parts
|
2017-06-19 16:10:18 +02:00
|
|
|
res = self.request("getFile", {"site": site, "inner_path": inner_path, "location": location, "file_size": file_size})
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
if not res or "body" not in res: # Error
|
2015-06-18 00:01:56 +02:00
|
|
|
return False
|
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
buff.write(res["body"])
|
|
|
|
res["body"] = None # Save memory
|
|
|
|
if res["location"] == res["size"]: # End of file
|
2015-06-18 00:01:56 +02:00
|
|
|
break
|
|
|
|
else:
|
2015-10-22 11:42:55 +02:00
|
|
|
location = res["location"]
|
2015-07-25 13:38:58 +02:00
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
self.download_bytes += res["location"]
|
2015-07-25 13:38:58 +02:00
|
|
|
self.download_time += (time.time() - s)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if self.site:
|
|
|
|
self.site.settings["bytes_recv"] = self.site.settings.get("bytes_recv", 0) + res["location"]
|
2015-07-25 13:38:58 +02:00
|
|
|
buff.seek(0)
|
|
|
|
return buff
|
|
|
|
|
|
|
|
# Download file out of msgpack context to save memory and cpu
|
|
|
|
def streamFile(self, site, inner_path):
|
|
|
|
location = 0
|
|
|
|
if config.use_tempfiles:
|
|
|
|
buff = tempfile.SpooledTemporaryFile(max_size=16 * 1024, mode='w+b')
|
|
|
|
else:
|
|
|
|
buff = StringIO()
|
|
|
|
|
|
|
|
s = time.time()
|
|
|
|
while True: # Read in 512k parts
|
2015-10-22 11:42:55 +02:00
|
|
|
res = self.request("streamFile", {"site": site, "inner_path": inner_path, "location": location}, stream_to=buff)
|
2015-07-25 13:38:58 +02:00
|
|
|
|
2016-03-26 00:22:46 +01:00
|
|
|
if not res or "location" not in res: # Error
|
2015-10-22 11:42:55 +02:00
|
|
|
self.log("Invalid response: %s" % res)
|
2015-07-25 13:38:58 +02:00
|
|
|
return False
|
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
if res["location"] == res["size"]: # End of file
|
2015-07-25 13:38:58 +02:00
|
|
|
break
|
|
|
|
else:
|
2015-10-22 11:42:55 +02:00
|
|
|
location = res["location"]
|
2015-07-25 13:38:58 +02:00
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
self.download_bytes += res["location"]
|
2015-06-18 00:01:56 +02:00
|
|
|
self.download_time += (time.time() - s)
|
2015-10-22 11:42:55 +02:00
|
|
|
self.site.settings["bytes_recv"] = self.site.settings.get("bytes_recv", 0) + res["location"]
|
2015-06-18 00:01:56 +02:00
|
|
|
buff.seek(0)
|
|
|
|
return buff
|
|
|
|
|
|
|
|
# Send a ping request
|
|
|
|
def ping(self):
|
|
|
|
response_time = None
|
2015-06-18 00:04:49 +02:00
|
|
|
for retry in range(1, 3): # Retry 3 times
|
2015-06-18 00:01:56 +02:00
|
|
|
s = time.time()
|
2015-06-18 00:06:41 +02:00
|
|
|
with gevent.Timeout(10.0, False): # 10 sec timeout, don't raise exception
|
2015-10-22 11:42:55 +02:00
|
|
|
res = self.request("ping")
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
if res and "body" in res and res["body"] == "Pong!":
|
2015-07-12 20:36:46 +02:00
|
|
|
response_time = time.time() - s
|
2015-06-18 00:04:49 +02:00
|
|
|
break # All fine, exit from for loop
|
2015-06-18 00:01:56 +02:00
|
|
|
# Timeout reached or bad response
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Ping timeout")
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connect()
|
|
|
|
time.sleep(1)
|
|
|
|
|
|
|
|
if response_time:
|
|
|
|
self.log("Ping: %.3f" % response_time)
|
|
|
|
else:
|
|
|
|
self.log("Ping failed")
|
|
|
|
self.last_ping = response_time
|
|
|
|
return response_time
|
|
|
|
|
|
|
|
# Request peer exchange from peer
|
|
|
|
def pex(self, site=None, need_num=5):
|
|
|
|
if not site:
|
|
|
|
site = self.site # If no site defined request peers for this site
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
|
|
|
# give back 5 connectible peers
|
|
|
|
packed_peers = helper.packPeers(self.site.getConnectablePeers(5))
|
|
|
|
request = {"site": site.address, "peers": packed_peers["ip4"], "need": need_num}
|
|
|
|
if packed_peers["onion"]:
|
|
|
|
request["peers_onion"] = packed_peers["onion"]
|
|
|
|
res = self.request("pex", request)
|
2015-10-22 11:42:55 +02:00
|
|
|
if not res or "error" in res:
|
2015-06-18 00:01:56 +02:00
|
|
|
return False
|
|
|
|
added = 0
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# Ip4
|
2015-10-22 11:42:55 +02:00
|
|
|
for peer in res.get("peers", []):
|
2015-09-27 02:08:53 +02:00
|
|
|
address = helper.unpackAddress(peer)
|
2015-06-18 00:01:56 +02:00
|
|
|
if site.addPeer(*address):
|
|
|
|
added += 1
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# Onion
|
|
|
|
for peer in res.get("peers_onion", []):
|
|
|
|
address = helper.unpackOnionAddress(peer)
|
|
|
|
if site.addPeer(*address):
|
|
|
|
added += 1
|
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
if added:
|
|
|
|
self.log("Added peers using pex: %s" % added)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
return added
|
|
|
|
|
|
|
|
# List modified files since the date
|
|
|
|
# Return: {inner_path: modification date,...}
|
|
|
|
def listModified(self, since):
|
|
|
|
return self.request("listModified", {"since": since, "site": self.site.address})
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
def updateHashfield(self, force=False):
|
|
|
|
# Don't update hashfield again in 15 min
|
2015-10-22 11:42:55 +02:00
|
|
|
if self.time_hashfield and time.time() - self.time_hashfield > 60 * 15 and not force:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return False
|
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_hashfield = time.time()
|
|
|
|
res = self.request("getHashfield", {"site": self.site.address})
|
|
|
|
if not res or "error" in res:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return False
|
2015-10-22 11:42:55 +02:00
|
|
|
self.hashfield.replaceFromString(res["hashfield_raw"])
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
|
|
|
return self.hashfield
|
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Find peers for hashids
|
2015-10-22 11:42:55 +02:00
|
|
|
# Return: {hash1: ["ip:port", "ip:port",...],...}
|
|
|
|
def findHashIds(self, hash_ids):
|
|
|
|
res = self.request("findHashIds", {"site": self.site.address, "hash_ids": hash_ids})
|
|
|
|
if not res or "error" in res:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return False
|
Rev957, Sidebar displays onion peers in graph, Sidebar display bad file retry number, Sidebar site Update/Pause/Delete, Ratelimit sidebar update, Encoded typo, Fix onion findHashId, More retry for bad files, Log file path errors, Testcase for self findhashIds, Testcase for Tor findHashId, Better Tor version parse, UiWebsocket callback on update/pause/resume/delete, Skip invalid postMessage messages
2016-03-09 00:48:57 +01:00
|
|
|
# Unpack IP4
|
2016-08-10 12:44:00 +02:00
|
|
|
back = {key: map(helper.unpackAddress, val) for key, val in res["peers"].items()[0:30]}
|
Rev957, Sidebar displays onion peers in graph, Sidebar display bad file retry number, Sidebar site Update/Pause/Delete, Ratelimit sidebar update, Encoded typo, Fix onion findHashId, More retry for bad files, Log file path errors, Testcase for self findhashIds, Testcase for Tor findHashId, Better Tor version parse, UiWebsocket callback on update/pause/resume/delete, Skip invalid postMessage messages
2016-03-09 00:48:57 +01:00
|
|
|
# Unpack onion
|
2016-08-10 12:44:00 +02:00
|
|
|
for hash, onion_peers in res.get("peers_onion", {}).items()[0:30]:
|
2017-02-27 00:07:51 +01:00
|
|
|
if hash not in back:
|
Rev957, Sidebar displays onion peers in graph, Sidebar display bad file retry number, Sidebar site Update/Pause/Delete, Ratelimit sidebar update, Encoded typo, Fix onion findHashId, More retry for bad files, Log file path errors, Testcase for self findhashIds, Testcase for Tor findHashId, Better Tor version parse, UiWebsocket callback on update/pause/resume/delete, Skip invalid postMessage messages
2016-03-09 00:48:57 +01:00
|
|
|
back[hash] = []
|
|
|
|
back[hash] += map(helper.unpackOnionAddress, onion_peers)
|
|
|
|
|
|
|
|
return back
|
2015-10-22 11:42:55 +02:00
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Send my hashfield to peer
|
|
|
|
# Return: True if sent
|
|
|
|
def sendMyHashfield(self):
|
|
|
|
if self.connection and self.connection.handshake.get("rev", 0) < 510:
|
|
|
|
return False # Not supported
|
|
|
|
if self.time_my_hashfield_sent and self.site.content_manager.hashfield.time_changed <= self.time_my_hashfield_sent:
|
|
|
|
return False # Peer already has the latest hashfield
|
|
|
|
|
|
|
|
res = self.request("setHashfield", {"site": self.site.address, "hashfield_raw": self.site.content_manager.hashfield.tostring()})
|
|
|
|
if not res or "error" in res:
|
|
|
|
return False
|
|
|
|
else:
|
|
|
|
self.time_my_hashfield_sent = time.time()
|
|
|
|
return True
|
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
# Stop and remove from site
|
2017-02-27 00:02:24 +01:00
|
|
|
def remove(self, reason="Removing"):
|
2015-10-22 11:42:55 +02:00
|
|
|
self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed))
|
|
|
|
if self.site and self.key in self.site.peers:
|
|
|
|
del(self.site.peers[self.key])
|
|
|
|
if self.connection:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.connection.close(reason)
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
# - EVENTS -
|
|
|
|
|
|
|
|
# On connection error
|
2017-02-27 00:02:24 +01:00
|
|
|
def onConnectionError(self, reason="Unknown"):
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection_error += 1
|
2017-06-19 16:13:06 +02:00
|
|
|
if len(self.site.peers) > 200:
|
|
|
|
limit = 3
|
|
|
|
else:
|
|
|
|
limit = 6
|
|
|
|
if self.connection_error >= limit: # Dead peer
|
2017-02-27 00:02:24 +01:00
|
|
|
self.remove("Peer connection: %s" % reason)
|
2015-06-18 00:01:56 +02:00
|
|
|
|
|
|
|
# Done working with peer
|
|
|
|
def onWorkerDone(self):
|
|
|
|
pass
|