version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
import os , logging , gevent , time , msgpack , sys
2015-01-12 02:03:45 +01:00
import zmq . green as zmq
from cStringIO import StringIO
from Config import config
2015-01-17 18:50:56 +01:00
from Debug import Debug
2015-01-12 02:03:45 +01:00
# Communicate remote peers
class Peer :
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
def __init__ ( self , ip , port , site = None ) :
2015-01-12 02:03:45 +01:00
self . ip = ip
self . port = port
2015-01-13 21:19:40 +01:00
self . site = site
2015-01-14 02:41:13 +01:00
self . key = " %s : %s " % ( ip , port )
2015-01-18 22:52:19 +01:00
self . log = None
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
self . connection_server = sys . modules [ " main " ] . file_server
2015-01-14 02:41:13 +01:00
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
self . connection = None
2015-01-18 22:52:19 +01:00
self . last_found = None # Time of last found in the torrent tracker
self . last_response = None # Time of last successfull response from peer
self . last_ping = None # Last response time for ping
2015-01-12 02:03:45 +01:00
self . added = time . time ( )
2015-01-18 22:52:19 +01:00
self . connection_error = 0 # Series of connection error
self . hash_failed = 0 # Number of bad files from peer
self . download_bytes = 0 # Bytes downloaded
self . download_time = 0 # Time spent to download
2015-01-12 02:03:45 +01:00
# Connect to host
def connect ( self ) :
2015-02-25 03:22:10 +01:00
if not self . log : self . log = logging . getLogger ( " Peer: %s : %s %s " % ( self . ip , self . port , self . site . address_short ) )
if self . connection :
self . log . debug ( " Getting connection (Closing %s )... " % self . connection )
self . connection . close ( )
else :
self . log . debug ( " Getting connection... " )
2015-03-06 02:31:51 +01:00
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
self . connection = None
2015-01-18 22:52:19 +01:00
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
try :
2015-02-25 03:22:10 +01:00
self . connection = self . connection_server . getConnection ( self . ip , self . port )
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
except Exception , err :
2015-03-11 01:12:53 +01:00
self . log . debug ( " Getting connection error: %s (connection_error: %s , hash_failed: %s ) " % ( Debug . formatException ( err ) , self . connection_error , self . hash_failed ) )
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
def __str__ ( self ) :
return " Peer %-12s " % self . ip
2015-01-12 02:03:45 +01:00
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
def __repr__ ( self ) :
return " < %s > " % self . __str__ ( )
2015-01-12 02:03:45 +01:00
# Found a peer on tracker
def found ( self ) :
self . last_found = time . time ( )
# Send a command to peer
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
def request ( self , cmd , params = { } ) :
if not self . connection or self . connection . closed :
self . connect ( )
2015-03-11 01:12:53 +01:00
if not self . connection :
self . onConnectionError ( )
return None # Connection failed
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
2015-03-06 02:31:51 +01:00
#if cmd != "ping" and self.last_response and time.time() - self.last_response > 20*60: # If last response if older than 20 minute, ping first to see if still alive
# if not self.ping(): return None
2015-01-18 22:52:19 +01:00
for retry in range ( 1 , 3 ) : # Retry 3 times
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
#if config.debug_socket: self.log.debug("sendCmd: %s %s" % (cmd, params.get("inner_path")))
2015-01-17 18:50:56 +01:00
try :
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
response = self . connection . request ( cmd , params )
if not response : raise Exception ( " Send error " )
#if config.debug_socket: self.log.debug("Got response to: %s" % cmd)
2015-01-17 18:50:56 +01:00
if " error " in response :
self . log . debug ( " %s error: %s " % ( cmd , response [ " error " ] ) )
self . onConnectionError ( )
else : # Successful request, reset connection error num
self . connection_error = 0
2015-01-18 22:52:19 +01:00
self . last_response = time . time ( )
2015-01-17 18:50:56 +01:00
return response
except Exception , err :
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
if type ( err ) . __name__ == " Notify " : # Greenlet kill by worker
self . log . debug ( " Peer worker got killed: %s , aborting cmd: %s " % ( err . message , cmd ) )
2015-01-17 18:50:56 +01:00
break
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
else :
self . onConnectionError ( )
self . log . debug ( " %s (connection_error: %s , hash_failed: %s , retry: %s ) " % ( Debug . formatException ( err ) , self . connection_error , self . hash_failed , retry ) )
time . sleep ( 1 * retry )
self . connect ( )
2015-01-17 18:50:56 +01:00
return None # Failed after 4 retry
2015-01-12 02:03:45 +01:00
# Get a file content from peer
def getFile ( self , site , inner_path ) :
location = 0
buff = StringIO ( )
s = time . time ( )
while 1 : # Read in 512k parts
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
back = self . request ( " getFile " , { " site " : site , " inner_path " : inner_path , " location " : location } ) # Get file content from last location
2015-01-13 21:19:40 +01:00
if not back or " body " not in back : # Error
2015-01-12 02:03:45 +01:00
return False
buff . write ( back [ " body " ] )
if back [ " location " ] == back [ " size " ] : # End of file
break
else :
location = back [ " location " ]
self . download_bytes + = back [ " location " ]
self . download_time + = ( time . time ( ) - s )
buff . seek ( 0 )
return buff
# Send a ping request
def ping ( self ) :
2015-01-18 22:52:19 +01:00
response_time = None
for retry in range ( 1 , 3 ) : # Retry 3 times
s = time . time ( )
with gevent . Timeout ( 10.0 , False ) : # 10 sec timeout, dont raise exception
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
response = self . request ( " ping " )
2015-01-18 22:52:19 +01:00
if response and " body " in response and response [ " body " ] == " Pong! " :
response_time = time . time ( ) - s
break # All fine, exit from for loop
# Timeout reached or bad response
self . onConnectionError ( )
2015-02-25 03:22:10 +01:00
self . connect ( )
2015-01-18 22:52:19 +01:00
time . sleep ( 1 )
if response_time :
self . log . debug ( " Ping: %.3f " % response_time )
else :
self . log . debug ( " Ping failed " )
self . last_ping = response_time
return response_time
2015-01-13 21:19:40 +01:00
# Stop and remove from site
def remove ( self ) :
self . log . debug ( " Removing peer...Connection error: %s , Hash failed: %s " % ( self . connection_error , self . hash_failed ) )
2015-01-27 00:21:44 +01:00
if self . key in self . site . peers : del ( self . site . peers [ self . key ] )
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
if self . connection :
self . connection . close ( )
2015-01-13 21:19:40 +01:00
# - EVENTS -
# On connection error
def onConnectionError ( self ) :
self . connection_error + = 1
2015-03-06 02:31:51 +01:00
if self . connection_error > = 3 : # Dead peer
2015-01-13 21:19:40 +01:00
self . remove ( )
# Done working with peer
def onWorkerDone ( self ) :
pass