2015-07-12 20:36:46 +02:00
|
|
|
import logging
|
|
|
|
import time
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
import sys
|
2019-01-20 16:26:04 +01:00
|
|
|
import itertools
|
2019-02-24 02:28:42 +01:00
|
|
|
import collections
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
|
|
|
import gevent
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
import io
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
2015-07-25 13:38:58 +02:00
|
|
|
from Config import config
|
2015-09-27 02:08:53 +02:00
|
|
|
from util import helper
|
2019-03-15 21:06:59 +01:00
|
|
|
from .PeerHashfield import PeerHashfield
|
2017-10-03 14:56:42 +02:00
|
|
|
from Plugin import PluginManager
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-25 13:38:58 +02:00
|
|
|
if config.use_tempfiles:
|
|
|
|
import tempfile
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# Communicate remote peers
|
2017-10-03 14:56:42 +02:00
|
|
|
@PluginManager.acceptPlugins
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
class Peer(object):
|
2015-07-25 13:38:58 +02:00
|
|
|
__slots__ = (
|
2019-05-30 04:26:41 +02:00
|
|
|
"ip", "port", "site", "key", "connection", "connection_server", "time_found", "time_response", "time_hashfield",
|
|
|
|
"time_added", "has_hashfield", "is_tracker_connection", "time_my_hashfield_sent", "last_ping", "reputation",
|
|
|
|
"last_content_json_update", "hashfield", "connection_error", "hash_failed", "download_bytes", "download_time"
|
2015-07-25 13:38:58 +02:00
|
|
|
)
|
2015-06-18 00:01:56 +02:00
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
def __init__(self, ip, port, site=None, connection_server=None):
|
2015-06-18 00:01:56 +02:00
|
|
|
self.ip = ip
|
|
|
|
self.port = port
|
|
|
|
self.site = site
|
|
|
|
self.key = "%s:%s" % (ip, port)
|
|
|
|
|
|
|
|
self.connection = None
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
self.connection_server = connection_server
|
2016-11-07 22:34:46 +01:00
|
|
|
self.has_hashfield = False # Lazy hashfield object not created yet
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
self.time_hashfield = None # Last time peer's hashfiled downloaded
|
|
|
|
self.time_my_hashfield_sent = None # Last time my hashfield sent to peer
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_found = time.time() # Time of last found in the torrent tracker
|
|
|
|
self.time_response = None # Time of last successful response from peer
|
|
|
|
self.time_added = time.time()
|
2015-06-18 00:03:11 +02:00
|
|
|
self.last_ping = None # Last response time for ping
|
2018-08-26 02:52:23 +02:00
|
|
|
self.is_tracker_connection = False # Tracker connection instead of normal peer
|
2017-02-27 00:06:13 +01:00
|
|
|
self.reputation = 0 # More likely to connect if larger
|
2016-03-30 23:05:43 +02:00
|
|
|
self.last_content_json_update = 0.0 # Modify date of last received content.json
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2015-06-18 00:03:11 +02:00
|
|
|
self.connection_error = 0 # Series of connection error
|
|
|
|
self.hash_failed = 0 # Number of bad files from peer
|
|
|
|
self.download_bytes = 0 # Bytes downloaded
|
|
|
|
self.download_time = 0 # Time spent to download
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2016-09-07 17:41:33 +02:00
|
|
|
def __getattr__(self, key):
|
|
|
|
if key == "hashfield":
|
2016-11-07 22:34:46 +01:00
|
|
|
self.has_hashfield = True
|
2016-09-07 17:41:33 +02:00
|
|
|
self.hashfield = PeerHashfield()
|
|
|
|
return self.hashfield
|
|
|
|
else:
|
|
|
|
return getattr(self, key)
|
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
def log(self, text):
|
2016-03-06 00:55:50 +01:00
|
|
|
if not config.verbose:
|
|
|
|
return # Only log if we are in debug mode
|
2015-06-18 00:01:56 +02:00
|
|
|
if self.site:
|
|
|
|
self.site.log.debug("%s:%s %s" % (self.ip, self.port, text))
|
|
|
|
else:
|
|
|
|
logging.debug("%s:%s %s" % (self.ip, self.port, text))
|
|
|
|
|
|
|
|
# Connect to host
|
|
|
|
def connect(self, connection=None):
|
2018-09-02 02:21:38 +02:00
|
|
|
if self.reputation < -10:
|
|
|
|
self.reputation = -10
|
|
|
|
if self.reputation > 10:
|
|
|
|
self.reputation = 10
|
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
if self.connection:
|
|
|
|
self.log("Getting connection (Closing %s)..." % self.connection)
|
2017-02-27 00:02:24 +01:00
|
|
|
self.connection.close("Connection change")
|
2015-06-18 00:01:56 +02:00
|
|
|
else:
|
2018-09-02 02:21:48 +02:00
|
|
|
self.log("Getting connection (reputation: %s)..." % self.reputation)
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2015-06-18 00:03:11 +02:00
|
|
|
if connection: # Connection specified
|
2018-01-30 13:58:01 +01:00
|
|
|
self.log("Assigning connection %s" % connection)
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection = connection
|
2016-04-25 02:23:06 +02:00
|
|
|
self.connection.sites += 1
|
2015-06-18 00:01:56 +02:00
|
|
|
else: # Try to find from connection pool or create new connection
|
|
|
|
self.connection = None
|
|
|
|
|
|
|
|
try:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if self.connection_server:
|
2018-08-26 02:52:23 +02:00
|
|
|
connection_server = self.connection_server
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
elif self.site:
|
2018-08-26 02:52:23 +02:00
|
|
|
connection_server = self.site.connection_server
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
else:
|
2019-04-15 12:31:33 +02:00
|
|
|
import main
|
|
|
|
connection_server = main.file_server
|
2018-08-26 02:52:23 +02:00
|
|
|
self.connection = connection_server.getConnection(self.ip, self.port, site=self.site, is_tracker_connection=self.is_tracker_connection)
|
2018-09-02 02:22:21 +02:00
|
|
|
self.reputation += 1
|
2016-04-25 02:23:06 +02:00
|
|
|
self.connection.sites += 1
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Getting connection error")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
|
|
|
|
(Debug.formatException(err), self.connection_error, self.hash_failed))
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection = None
|
2018-08-26 02:52:23 +02:00
|
|
|
return self.connection
|
2015-06-18 00:01:56 +02:00
|
|
|
|
|
|
|
# Check if we have connection to peer
|
|
|
|
def findConnection(self):
|
2015-06-18 00:03:11 +02:00
|
|
|
if self.connection and self.connection.connected: # We have connection to peer
|
2015-06-18 00:01:56 +02:00
|
|
|
return self.connection
|
2015-06-18 00:03:11 +02:00
|
|
|
else: # Try to find from other sites connections
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False, site=self.site)
|
2016-05-05 12:12:18 +02:00
|
|
|
if self.connection:
|
|
|
|
self.connection.sites += 1
|
2015-06-18 00:01:56 +02:00
|
|
|
return self.connection
|
|
|
|
|
|
|
|
def __str__(self):
|
2019-11-25 14:31:12 +01:00
|
|
|
if self.site:
|
|
|
|
return "Peer:%-12s of %s" % (self.ip, self.site.address_short)
|
|
|
|
else:
|
|
|
|
return "Peer:%-12s" % self.ip
|
2015-06-18 00:01:56 +02:00
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return "<%s>" % self.__str__()
|
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
def packMyAddress(self):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if self.ip.endswith(".onion"):
|
|
|
|
return helper.packOnionAddress(self.ip, self.port)
|
|
|
|
else:
|
|
|
|
return helper.packAddress(self.ip, self.port)
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2018-02-08 17:53:17 +01:00
|
|
|
# Found a peer from a source
|
|
|
|
def found(self, source="other"):
|
2018-09-04 15:43:20 +02:00
|
|
|
if self.reputation < 5:
|
|
|
|
if source == "tracker":
|
2019-11-25 14:30:51 +01:00
|
|
|
if self.ip.endswith(".onion"):
|
|
|
|
self.reputation += 1
|
|
|
|
else:
|
|
|
|
self.reputation += 2
|
2018-09-04 15:43:20 +02:00
|
|
|
elif source == "local":
|
2019-11-25 14:32:06 +01:00
|
|
|
self.reputation += 20
|
2018-09-04 15:43:20 +02:00
|
|
|
|
2018-02-08 17:53:17 +01:00
|
|
|
if source in ("tracker", "local"):
|
|
|
|
self.site.peers_recent.appendleft(self)
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_found = time.time()
|
2015-06-18 00:01:56 +02:00
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Send a command to peer and return response value
|
2015-07-25 13:38:58 +02:00
|
|
|
def request(self, cmd, params={}, stream_to=None):
|
2015-06-18 00:01:56 +02:00
|
|
|
if not self.connection or self.connection.closed:
|
|
|
|
self.connect()
|
|
|
|
if not self.connection:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Reconnect error")
|
2015-07-12 20:36:46 +02:00
|
|
|
return None # Connection failed
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2017-10-03 15:23:03 +02:00
|
|
|
self.log("Send request: %s %s %s %s" % (params.get("site", ""), cmd, params.get("inner_path", ""), params.get("location", "")))
|
2016-03-03 21:12:16 +01:00
|
|
|
|
2015-11-05 23:19:36 +01:00
|
|
|
for retry in range(1, 4): # Retry 3 times
|
2015-06-18 00:01:56 +02:00
|
|
|
try:
|
2017-06-13 14:12:48 +02:00
|
|
|
if not self.connection:
|
|
|
|
raise Exception("No connection found")
|
2015-10-22 11:42:55 +02:00
|
|
|
res = self.connection.request(cmd, params, stream_to)
|
|
|
|
if not res:
|
2015-06-18 00:01:56 +02:00
|
|
|
raise Exception("Send error")
|
2015-10-22 11:42:55 +02:00
|
|
|
if "error" in res:
|
|
|
|
self.log("%s error: %s" % (cmd, res["error"]))
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Response error")
|
|
|
|
break
|
2015-07-12 20:36:46 +02:00
|
|
|
else: # Successful request, reset connection error num
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection_error = 0
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_response = time.time()
|
2017-10-03 15:23:25 +02:00
|
|
|
if res:
|
|
|
|
return res
|
|
|
|
else:
|
|
|
|
raise Exception("Invalid response: %s" % res)
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2015-06-18 00:03:11 +02:00
|
|
|
if type(err).__name__ == "Notify": # Greenlet killed by worker
|
2015-06-18 00:01:56 +02:00
|
|
|
self.log("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd))
|
|
|
|
break
|
|
|
|
else:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Request error")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log(
|
|
|
|
"%s (connection_error: %s, hash_failed: %s, retry: %s)" %
|
|
|
|
(Debug.formatException(err), self.connection_error, self.hash_failed, retry)
|
|
|
|
)
|
|
|
|
time.sleep(1 * retry)
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connect()
|
|
|
|
return None # Failed after 4 retry
|
|
|
|
|
|
|
|
# Get a file content from peer
|
2017-10-03 15:26:39 +02:00
|
|
|
def getFile(self, site, inner_path, file_size=None, pos_from=0, pos_to=None, streaming=False):
|
|
|
|
if file_size and file_size > 5 * 1024 * 1024:
|
|
|
|
max_read_size = 1024 * 1024
|
2015-07-25 13:38:58 +02:00
|
|
|
else:
|
2017-10-03 15:26:39 +02:00
|
|
|
max_read_size = 512 * 1024
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-10-03 15:26:39 +02:00
|
|
|
if pos_to:
|
|
|
|
read_bytes = min(max_read_size, pos_to - pos_from)
|
|
|
|
else:
|
|
|
|
read_bytes = max_read_size
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2017-10-03 15:26:39 +02:00
|
|
|
location = pos_from
|
2015-07-25 13:38:58 +02:00
|
|
|
|
|
|
|
if config.use_tempfiles:
|
|
|
|
buff = tempfile.SpooledTemporaryFile(max_size=16 * 1024, mode='w+b')
|
|
|
|
else:
|
2019-03-15 21:06:59 +01:00
|
|
|
buff = io.BytesIO()
|
2015-07-25 13:38:58 +02:00
|
|
|
|
|
|
|
s = time.time()
|
2017-10-03 15:26:39 +02:00
|
|
|
while True: # Read in smaller parts
|
|
|
|
if config.stream_downloads or read_bytes > 256 * 1024 or streaming:
|
|
|
|
res = self.request("streamFile", {"site": site, "inner_path": inner_path, "location": location, "read_bytes": read_bytes, "file_size": file_size}, stream_to=buff)
|
|
|
|
if not res or "location" not in res: # Error
|
|
|
|
return False
|
|
|
|
else:
|
|
|
|
self.log("Send: %s" % inner_path)
|
|
|
|
res = self.request("getFile", {"site": site, "inner_path": inner_path, "location": location, "read_bytes": read_bytes, "file_size": file_size})
|
|
|
|
if not res or "location" not in res: # Error
|
|
|
|
return False
|
|
|
|
self.log("Recv: %s" % inner_path)
|
|
|
|
buff.write(res["body"])
|
|
|
|
res["body"] = None # Save memory
|
|
|
|
|
|
|
|
if res["location"] == res["size"] or res["location"] == pos_to: # End of file
|
2015-07-25 13:38:58 +02:00
|
|
|
break
|
|
|
|
else:
|
2015-10-22 11:42:55 +02:00
|
|
|
location = res["location"]
|
2017-10-03 15:26:39 +02:00
|
|
|
if pos_to:
|
|
|
|
read_bytes = min(max_read_size, pos_to - location)
|
2015-07-25 13:38:58 +02:00
|
|
|
|
2017-10-03 15:26:39 +02:00
|
|
|
if pos_to:
|
|
|
|
recv = pos_to - pos_from
|
|
|
|
else:
|
|
|
|
recv = res["location"]
|
|
|
|
|
|
|
|
self.download_bytes += recv
|
2015-06-18 00:01:56 +02:00
|
|
|
self.download_time += (time.time() - s)
|
2017-10-03 15:26:39 +02:00
|
|
|
if self.site:
|
|
|
|
self.site.settings["bytes_recv"] = self.site.settings.get("bytes_recv", 0) + recv
|
|
|
|
self.log("Downloaded: %s, pos: %s, read_bytes: %s" % (inner_path, buff.tell(), read_bytes))
|
2015-06-18 00:01:56 +02:00
|
|
|
buff.seek(0)
|
|
|
|
return buff
|
|
|
|
|
|
|
|
# Send a ping request
|
|
|
|
def ping(self):
|
|
|
|
response_time = None
|
2015-06-18 00:04:49 +02:00
|
|
|
for retry in range(1, 3): # Retry 3 times
|
2015-06-18 00:01:56 +02:00
|
|
|
s = time.time()
|
2015-06-18 00:06:41 +02:00
|
|
|
with gevent.Timeout(10.0, False): # 10 sec timeout, don't raise exception
|
2015-10-22 11:42:55 +02:00
|
|
|
res = self.request("ping")
|
2015-06-18 00:01:56 +02:00
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
if res and "body" in res and res["body"] == b"Pong!":
|
2015-07-12 20:36:46 +02:00
|
|
|
response_time = time.time() - s
|
2015-06-18 00:04:49 +02:00
|
|
|
break # All fine, exit from for loop
|
2015-06-18 00:01:56 +02:00
|
|
|
# Timeout reached or bad response
|
2017-02-27 00:02:24 +01:00
|
|
|
self.onConnectionError("Ping timeout")
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connect()
|
|
|
|
time.sleep(1)
|
|
|
|
|
|
|
|
if response_time:
|
|
|
|
self.log("Ping: %.3f" % response_time)
|
|
|
|
else:
|
|
|
|
self.log("Ping failed")
|
|
|
|
self.last_ping = response_time
|
|
|
|
return response_time
|
|
|
|
|
|
|
|
# Request peer exchange from peer
|
|
|
|
def pex(self, site=None, need_num=5):
|
|
|
|
if not site:
|
|
|
|
site = self.site # If no site defined request peers for this site
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
|
|
|
# give back 5 connectible peers
|
2018-01-30 13:58:01 +01:00
|
|
|
packed_peers = helper.packPeers(self.site.getConnectablePeers(5, allow_private=False))
|
2019-01-20 16:26:04 +01:00
|
|
|
request = {"site": site.address, "peers": packed_peers["ipv4"], "need": need_num}
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if packed_peers["onion"]:
|
|
|
|
request["peers_onion"] = packed_peers["onion"]
|
2019-01-20 16:26:04 +01:00
|
|
|
if packed_peers["ipv6"]:
|
|
|
|
request["peers_ipv6"] = packed_peers["ipv6"]
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
res = self.request("pex", request)
|
2015-10-22 11:42:55 +02:00
|
|
|
if not res or "error" in res:
|
2015-06-18 00:01:56 +02:00
|
|
|
return False
|
|
|
|
added = 0
|
2019-01-20 16:26:04 +01:00
|
|
|
|
2019-02-24 02:29:08 +01:00
|
|
|
# Remove unsupported peer types
|
2019-05-02 17:17:57 +02:00
|
|
|
if "peers_ipv6" in res and self.connection and "ipv6" not in self.connection.server.supported_ip_types:
|
2019-02-24 02:29:08 +01:00
|
|
|
del res["peers_ipv6"]
|
|
|
|
|
2019-05-02 17:17:57 +02:00
|
|
|
if "peers_onion" in res and self.connection and "onion" not in self.connection.server.supported_ip_types:
|
2019-02-24 02:29:08 +01:00
|
|
|
del res["peers_onion"]
|
|
|
|
|
|
|
|
# Add IPv4 + IPv6
|
2019-01-20 16:26:04 +01:00
|
|
|
for peer in itertools.chain(res.get("peers", []), res.get("peers_ipv6", [])):
|
2015-09-27 02:08:53 +02:00
|
|
|
address = helper.unpackAddress(peer)
|
2018-02-08 17:57:26 +01:00
|
|
|
if site.addPeer(*address, source="pex"):
|
2015-06-18 00:01:56 +02:00
|
|
|
added += 1
|
2019-02-24 02:29:08 +01:00
|
|
|
|
|
|
|
# Add Onion
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
for peer in res.get("peers_onion", []):
|
|
|
|
address = helper.unpackOnionAddress(peer)
|
2018-02-08 17:57:26 +01:00
|
|
|
if site.addPeer(*address, source="pex"):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
added += 1
|
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
if added:
|
|
|
|
self.log("Added peers using pex: %s" % added)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
return added
|
|
|
|
|
|
|
|
# List modified files since the date
|
|
|
|
# Return: {inner_path: modification date,...}
|
|
|
|
def listModified(self, since):
|
|
|
|
return self.request("listModified", {"since": since, "site": self.site.address})
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
def updateHashfield(self, force=False):
|
2017-10-03 15:26:53 +02:00
|
|
|
# Don't update hashfield again in 5 min
|
|
|
|
if self.time_hashfield and time.time() - self.time_hashfield < 5 * 60 and not force:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return False
|
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_hashfield = time.time()
|
|
|
|
res = self.request("getHashfield", {"site": self.site.address})
|
2019-01-20 16:26:17 +01:00
|
|
|
if not res or "error" in res or "hashfield_raw" not in res:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return False
|
2019-03-16 00:42:21 +01:00
|
|
|
self.hashfield.replaceFromBytes(res["hashfield_raw"])
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
|
|
|
return self.hashfield
|
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Find peers for hashids
|
2015-10-22 11:42:55 +02:00
|
|
|
# Return: {hash1: ["ip:port", "ip:port",...],...}
|
|
|
|
def findHashIds(self, hash_ids):
|
|
|
|
res = self.request("findHashIds", {"site": self.site.address, "hash_ids": hash_ids})
|
2018-01-19 02:20:20 +01:00
|
|
|
if not res or "error" in res or type(res) is not dict:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return False
|
2019-01-20 16:27:12 +01:00
|
|
|
|
2019-02-24 02:28:42 +01:00
|
|
|
back = collections.defaultdict(list)
|
2019-01-20 16:27:12 +01:00
|
|
|
|
|
|
|
for ip_type in ["ipv4", "ipv6", "onion"]:
|
|
|
|
if ip_type == "ipv4":
|
|
|
|
key = "peers"
|
|
|
|
else:
|
|
|
|
key = "peers_%s" % ip_type
|
2019-03-15 21:06:59 +01:00
|
|
|
for hash, peers in list(res.get(key, {}).items())[0:30]:
|
2019-01-20 16:27:12 +01:00
|
|
|
if ip_type == "onion":
|
|
|
|
unpacker_func = helper.unpackOnionAddress
|
|
|
|
else:
|
|
|
|
unpacker_func = helper.unpackAddress
|
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
back[hash] += list(map(unpacker_func, peers))
|
Rev957, Sidebar displays onion peers in graph, Sidebar display bad file retry number, Sidebar site Update/Pause/Delete, Ratelimit sidebar update, Encoded typo, Fix onion findHashId, More retry for bad files, Log file path errors, Testcase for self findhashIds, Testcase for Tor findHashId, Better Tor version parse, UiWebsocket callback on update/pause/resume/delete, Skip invalid postMessage messages
2016-03-09 00:48:57 +01:00
|
|
|
|
2019-01-24 15:18:12 +01:00
|
|
|
for hash in res.get("my", []):
|
2020-01-28 16:57:20 +01:00
|
|
|
if self.connection:
|
|
|
|
back[hash].append((self.connection.ip, self.connection.port))
|
|
|
|
else:
|
|
|
|
back[hash].append((self.ip, self.port))
|
2019-01-24 15:18:12 +01:00
|
|
|
|
Rev957, Sidebar displays onion peers in graph, Sidebar display bad file retry number, Sidebar site Update/Pause/Delete, Ratelimit sidebar update, Encoded typo, Fix onion findHashId, More retry for bad files, Log file path errors, Testcase for self findhashIds, Testcase for Tor findHashId, Better Tor version parse, UiWebsocket callback on update/pause/resume/delete, Skip invalid postMessage messages
2016-03-09 00:48:57 +01:00
|
|
|
return back
|
2015-10-22 11:42:55 +02:00
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Send my hashfield to peer
|
|
|
|
# Return: True if sent
|
|
|
|
def sendMyHashfield(self):
|
|
|
|
if self.connection and self.connection.handshake.get("rev", 0) < 510:
|
|
|
|
return False # Not supported
|
|
|
|
if self.time_my_hashfield_sent and self.site.content_manager.hashfield.time_changed <= self.time_my_hashfield_sent:
|
|
|
|
return False # Peer already has the latest hashfield
|
|
|
|
|
2019-03-16 00:42:21 +01:00
|
|
|
res = self.request("setHashfield", {"site": self.site.address, "hashfield_raw": self.site.content_manager.hashfield.tobytes()})
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
if not res or "error" in res:
|
|
|
|
return False
|
|
|
|
else:
|
|
|
|
self.time_my_hashfield_sent = time.time()
|
|
|
|
return True
|
|
|
|
|
2019-05-30 04:26:41 +02:00
|
|
|
def publish(self, address, inner_path, body, modified, diffs=[]):
|
|
|
|
if len(body) > 10 * 1024 and self.connection and self.connection.handshake.get("rev", 0) >= 4095:
|
|
|
|
# To save bw we don't push big content.json to peers
|
|
|
|
body = b""
|
|
|
|
|
|
|
|
return self.request("update", {
|
|
|
|
"site": address,
|
|
|
|
"inner_path": inner_path,
|
|
|
|
"body": body,
|
|
|
|
"modified": modified,
|
|
|
|
"diffs": diffs
|
|
|
|
})
|
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
# Stop and remove from site
|
2017-02-27 00:02:24 +01:00
|
|
|
def remove(self, reason="Removing"):
|
2015-10-22 11:42:55 +02:00
|
|
|
self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed))
|
|
|
|
if self.site and self.key in self.site.peers:
|
|
|
|
del(self.site.peers[self.key])
|
2018-02-08 18:07:09 +01:00
|
|
|
|
|
|
|
if self.site and self in self.site.peers_recent:
|
|
|
|
self.site.peers_recent.remove(self)
|
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
if self.connection:
|
2017-02-27 00:02:24 +01:00
|
|
|
self.connection.close(reason)
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
2015-06-18 00:01:56 +02:00
|
|
|
# - EVENTS -
|
|
|
|
|
|
|
|
# On connection error
|
2017-02-27 00:02:24 +01:00
|
|
|
def onConnectionError(self, reason="Unknown"):
|
2015-06-18 00:01:56 +02:00
|
|
|
self.connection_error += 1
|
2017-06-20 20:28:35 +02:00
|
|
|
if self.site and len(self.site.peers) > 200:
|
2017-06-19 16:13:06 +02:00
|
|
|
limit = 3
|
|
|
|
else:
|
|
|
|
limit = 6
|
2018-09-02 02:22:21 +02:00
|
|
|
self.reputation -= 1
|
2017-06-19 16:13:06 +02:00
|
|
|
if self.connection_error >= limit: # Dead peer
|
2017-02-27 00:02:24 +01:00
|
|
|
self.remove("Peer connection: %s" % reason)
|
2015-06-18 00:01:56 +02:00
|
|
|
|
|
|
|
# Done working with peer
|
|
|
|
def onWorkerDone(self):
|
|
|
|
pass
|