2019-07-03 19:24:50 +02:00
import random
2018-08-26 02:45:37 +02:00
import time
import os
import logging
import json
import atexit
2019-07-04 18:59:59 +02:00
import re
2018-08-26 02:45:37 +02:00
import gevent
from Config import config
from Plugin import PluginManager
from util import helper
class TrackerStorage ( object ) :
def __init__ ( self ) :
2019-07-03 04:48:07 +02:00
self . site_announcer = None
2018-08-26 02:45:37 +02:00
self . log = logging . getLogger ( " TrackerStorage " )
2019-07-03 08:51:39 +02:00
self . working_tracker_time_interval = 60 * 60
self . tracker_down_time_interval = 60 * 60
2019-07-03 13:19:02 +02:00
self . tracker_discover_time_interval = 5 * 60
2019-07-03 08:51:39 +02:00
2019-07-04 18:59:59 +02:00
self . working_shared_trackers_limit_per_protocol = { }
self . working_shared_trackers_limit_per_protocol [ " other " ] = 2
2018-08-26 02:45:37 +02:00
self . file_path = " %s /trackers.json " % config . data_dir
2019-01-20 19:12:11 +01:00
self . load ( )
2018-08-26 02:45:37 +02:00
self . time_discover = 0.0
2019-07-03 09:02:04 +02:00
self . time_success = 0.0
2018-08-26 02:45:37 +02:00
atexit . register ( self . save )
2019-07-04 18:59:59 +02:00
def initTrackerLimitForProtocol ( self ) :
for s in re . split ( " [,;] " , config . working_shared_trackers_limit_per_protocol ) :
x = s . split ( " = " , 1 )
if len ( x ) == 1 :
x = [ " other " , x [ 0 ] ]
try :
self . working_shared_trackers_limit_per_protocol [ x [ 0 ] ] = int ( x [ 1 ] )
except ValueError :
pass
self . log . debug ( " Limits per protocol: %s " % self . working_shared_trackers_limit_per_protocol )
def getTrackerLimitForProtocol ( self , protocol ) :
l = self . working_shared_trackers_limit_per_protocol
return l . get ( protocol , l . get ( " other " ) )
2019-07-03 04:48:07 +02:00
def setSiteAnnouncer ( self , site_announcer ) :
2019-07-04 18:59:59 +02:00
if self . site_announcer :
2019-07-03 04:48:07 +02:00
return
self . site_announcer = site_announcer
2019-07-04 18:59:59 +02:00
self . initTrackerLimitForProtocol ( )
2019-07-03 04:48:07 +02:00
self . recheckValidTrackers ( )
def isTrackerAddressValid ( self , tracker_address ) :
if not self . site_announcer : # Not completely initialized, skip check
return True
address_parts = self . site_announcer . getAddressParts ( tracker_address )
if not address_parts :
self . log . debug ( " Invalid tracker address: %s " % tracker_address )
return False
handler = self . site_announcer . getTrackerHandler ( address_parts [ " protocol " ] )
if not handler :
self . log . debug ( " Invalid tracker address: Unknown protocol %s : %s " % ( address_parts [ " protocol " ] , tracker_address ) )
return False
return True
def recheckValidTrackers ( self ) :
trackers = self . getTrackers ( )
for address , tracker in list ( trackers . items ( ) ) :
if not self . isTrackerAddressValid ( address ) :
del trackers [ address ]
2019-07-03 08:37:03 +02:00
def getNormalizedTrackerProtocol ( self , tracker_address ) :
if not self . site_announcer :
return None
address_parts = self . site_announcer . getAddressParts ( tracker_address )
if not address_parts :
return None
protocol = address_parts [ " protocol " ]
if protocol == " https " :
2019-07-03 11:16:16 +02:00
protocol = " http "
2019-07-03 08:37:03 +02:00
return protocol
def getSupportedProtocols ( self ) :
if not self . site_announcer :
return None
supported_trackers = self . site_announcer . getSupportedTrackers ( )
2019-07-03 19:11:46 +02:00
# If a tracker is in our list, but is absent from the results of getSupportedTrackers(),
# it seems to be supported by software, but forbidden by the settings or network configuration.
# We check and remove thoose trackers here, since onTrackerError() is never emitted for them.
trackers = self . getTrackers ( )
for tracker_address , tracker in list ( trackers . items ( ) ) :
t = max ( trackers [ tracker_address ] [ " time_added " ] ,
trackers [ tracker_address ] [ " time_success " ] )
if tracker_address not in supported_trackers and t < time . time ( ) - self . tracker_down_time_interval :
self . log . debug ( " Tracker %s looks unused, removing. " % tracker_address )
del trackers [ tracker_address ]
2019-07-03 13:14:38 +02:00
protocols = set ( )
2019-07-03 08:37:03 +02:00
for tracker_address in supported_trackers :
protocol = self . getNormalizedTrackerProtocol ( tracker_address )
if not protocol :
continue
2019-07-03 13:14:38 +02:00
protocols . add ( protocol )
2019-07-03 08:37:03 +02:00
2019-07-03 13:14:38 +02:00
protocols = list ( protocols )
2019-07-03 08:37:03 +02:00
self . log . debug ( " Supported tracker protocols: %s " % protocols )
return protocols
2018-08-26 02:45:37 +02:00
def getDefaultFile ( self ) :
return { " shared " : { } }
def onTrackerFound ( self , tracker_address , type = " shared " , my = False ) :
2019-07-03 04:48:07 +02:00
if not self . isTrackerAddressValid ( tracker_address ) :
2018-08-27 11:49:24 +02:00
return False
2019-07-02 19:03:55 +02:00
trackers = self . getTrackers ( type )
2018-08-29 19:51:26 +02:00
added = False
2018-08-26 02:45:37 +02:00
if tracker_address not in trackers :
trackers [ tracker_address ] = {
" time_added " : time . time ( ) ,
" time_success " : 0 ,
" latency " : 99.0 ,
" num_error " : 0 ,
" my " : False
}
2018-08-26 22:52:30 +02:00
self . log . debug ( " New tracker found: %s " % tracker_address )
2018-08-29 19:51:26 +02:00
added = True
2018-08-27 11:49:05 +02:00
2018-08-26 02:45:37 +02:00
trackers [ tracker_address ] [ " time_found " ] = time . time ( )
trackers [ tracker_address ] [ " my " ] = my
2018-08-29 19:51:26 +02:00
return added
2018-08-26 02:45:37 +02:00
def onTrackerSuccess ( self , tracker_address , latency ) :
trackers = self . getTrackers ( )
if tracker_address not in trackers :
return False
trackers [ tracker_address ] [ " latency " ] = latency
trackers [ tracker_address ] [ " time_success " ] = time . time ( )
trackers [ tracker_address ] [ " num_error " ] = 0
2019-07-03 09:02:04 +02:00
self . time_success = time . time ( )
2018-08-26 02:45:37 +02:00
def onTrackerError ( self , tracker_address ) :
trackers = self . getTrackers ( )
if tracker_address not in trackers :
return False
trackers [ tracker_address ] [ " time_error " ] = time . time ( )
trackers [ tracker_address ] [ " num_error " ] + = 1
2019-07-03 09:02:04 +02:00
if self . time_success < time . time ( ) - self . tracker_down_time_interval / 2 :
# Don't drop trackers from the list, if there haven't been any successful announces recently.
# There may be network connectivity issues.
return
2019-07-03 20:00:19 +02:00
protocol = self . getNormalizedTrackerProtocol ( tracker_address ) or " "
nr_working_trackers_for_protocol = len ( self . getTrackersPerProtocol ( working_only = True ) . get ( protocol , [ ] ) )
nr_working_trackers = len ( self . getWorkingTrackers ( ) )
error_limit = 30
2019-07-04 18:59:59 +02:00
if nr_working_trackers_for_protocol > = self . getTrackerLimitForProtocol ( protocol ) :
2019-07-03 20:00:19 +02:00
error_limit = 10
if nr_working_trackers > = config . working_shared_trackers_limit :
error_limit = 5
2018-09-02 02:17:18 +02:00
2019-07-03 08:51:39 +02:00
if trackers [ tracker_address ] [ " num_error " ] > error_limit and trackers [ tracker_address ] [ " time_success " ] < time . time ( ) - self . tracker_down_time_interval :
2018-08-26 02:45:37 +02:00
self . log . debug ( " Tracker %s looks down, removing. " % tracker_address )
del trackers [ tracker_address ]
2019-07-03 08:37:03 +02:00
def isTrackerWorking ( self , tracker_address , type = " shared " ) :
trackers = self . getTrackers ( type )
tracker = trackers [ tracker_address ]
if not tracker :
return False
2019-07-03 08:51:39 +02:00
if tracker [ " time_success " ] > time . time ( ) - self . working_tracker_time_interval :
2019-07-03 08:37:03 +02:00
return True
return False
2018-08-26 02:45:37 +02:00
def getTrackers ( self , type = " shared " ) :
return self . file_content . setdefault ( type , { } )
2019-07-03 08:37:03 +02:00
def getTrackersPerProtocol ( self , type = " shared " , working_only = False ) :
if not self . site_announcer :
return None
trackers = self . getTrackers ( type )
trackers_per_protocol = { }
for tracker_address in trackers :
protocol = self . getNormalizedTrackerProtocol ( tracker_address )
if not protocol :
continue
if not working_only or self . isTrackerWorking ( tracker_address , type = type ) :
trackers_per_protocol . setdefault ( protocol , [ ] ) . append ( tracker_address )
return trackers_per_protocol
2018-08-27 11:48:52 +02:00
def getWorkingTrackers ( self , type = " shared " ) :
2018-08-26 02:45:37 +02:00
trackers = {
2019-03-15 21:06:59 +01:00
key : tracker for key , tracker in self . getTrackers ( type ) . items ( )
2019-07-03 08:37:03 +02:00
if self . isTrackerWorking ( key , type )
2018-08-26 02:45:37 +02:00
}
return trackers
2019-01-20 19:12:11 +01:00
def getFileContent ( self ) :
2018-08-26 02:45:37 +02:00
if not os . path . isfile ( self . file_path ) :
open ( self . file_path , " w " ) . write ( " {} " )
return self . getDefaultFile ( )
try :
return json . load ( open ( self . file_path ) )
except Exception as err :
self . log . error ( " Error loading trackers list: %s " % err )
return self . getDefaultFile ( )
2019-01-20 19:12:11 +01:00
def load ( self ) :
self . file_content = self . getFileContent ( )
trackers = self . getTrackers ( )
self . log . debug ( " Loaded %s shared trackers " % len ( trackers ) )
2019-03-15 21:06:59 +01:00
for address , tracker in list ( trackers . items ( ) ) :
2019-01-20 19:12:11 +01:00
tracker [ " num_error " ] = 0
2019-07-03 04:48:07 +02:00
self . recheckValidTrackers ( )
2019-01-20 19:12:11 +01:00
2018-08-26 02:45:37 +02:00
def save ( self ) :
s = time . time ( )
2019-03-16 02:11:38 +01:00
helper . atomicWrite ( self . file_path , json . dumps ( self . file_content , indent = 2 , sort_keys = True ) . encode ( " utf8 " ) )
2018-08-26 02:45:37 +02:00
self . log . debug ( " Saved in %.3f s " % ( time . time ( ) - s ) )
2019-07-03 08:37:03 +02:00
def enoughWorkingTrackers ( self , type = " shared " ) :
supported_protocols = self . getSupportedProtocols ( )
if not supported_protocols :
return False
trackers_per_protocol = self . getTrackersPerProtocol ( type = " shared " , working_only = True )
if not trackers_per_protocol :
return False
total_nr = 0
for protocol in supported_protocols :
trackers = trackers_per_protocol . get ( protocol , [ ] )
2019-07-04 18:59:59 +02:00
if len ( trackers ) < self . getTrackerLimitForProtocol ( protocol ) :
2019-07-03 08:37:03 +02:00
self . log . debug ( " Not enough working trackers for protocol %s : %s < %s " % (
2019-07-04 18:59:59 +02:00
protocol , len ( trackers ) , self . getTrackerLimitForProtocol ( protocol ) ) )
2019-07-03 08:37:03 +02:00
return False
total_nr + = len ( trackers )
if total_nr < config . working_shared_trackers_limit :
self . log . debug ( " Not enough working trackers: %s < %s " % (
total_nr , config . working_shared_trackers_limit ) )
return False
return True
2018-08-26 02:45:37 +02:00
def discoverTrackers ( self , peers ) :
2019-07-03 12:45:01 +02:00
if self . enoughWorkingTrackers ( type = " shared " ) :
2018-08-27 11:48:39 +02:00
return False
2019-07-03 12:45:01 +02:00
self . log . debug ( " Discovering trackers from %s peers... " % len ( peers ) )
2018-08-26 02:45:37 +02:00
s = time . time ( )
num_success = 0
for peer in peers :
if peer . connection and peer . connection . handshake . get ( " rev " , 0 ) < 3560 :
continue # Not supported
res = peer . request ( " getTrackers " )
if not res or " error " in res :
continue
num_success + = 1
2019-07-03 19:24:50 +02:00
random . shuffle ( res [ " trackers " ] )
2018-08-26 02:45:37 +02:00
for tracker_address in res [ " trackers " ] :
2019-03-16 02:11:22 +01:00
if type ( tracker_address ) is bytes : # Backward compatibilitys
2019-03-16 03:44:01 +01:00
tracker_address = tracker_address . decode ( " utf8 " )
2018-08-27 11:48:25 +02:00
added = self . onTrackerFound ( tracker_address )
if added : # Only add one tracker from one source
break
2018-08-26 02:45:37 +02:00
if not num_success and len ( peers ) < 20 :
self . time_discover = 0.0
if num_success :
self . save ( )
2018-09-17 15:20:02 +02:00
self . log . debug ( " Trackers discovered from %s / %s peers in %.3f s " % ( num_success , len ( peers ) , time . time ( ) - s ) )
2018-08-26 02:45:37 +02:00
2018-08-27 11:48:11 +02:00
if " tracker_storage " not in locals ( ) :
tracker_storage = TrackerStorage ( )
2018-08-26 02:45:37 +02:00
@PluginManager.registerTo ( " SiteAnnouncer " )
class SiteAnnouncerPlugin ( object ) :
def getTrackers ( self ) :
2019-07-03 04:48:07 +02:00
tracker_storage . setSiteAnnouncer ( self )
2019-07-03 13:19:02 +02:00
if tracker_storage . time_discover < time . time ( ) - tracker_storage . tracker_discover_time_interval :
2018-08-26 02:45:37 +02:00
tracker_storage . time_discover = time . time ( )
gevent . spawn ( tracker_storage . discoverTrackers , self . site . getConnectedPeers ( ) )
trackers = super ( SiteAnnouncerPlugin , self ) . getTrackers ( )
2019-03-15 21:06:59 +01:00
shared_trackers = list ( tracker_storage . getTrackers ( " shared " ) . keys ( ) )
2018-08-26 02:45:37 +02:00
if shared_trackers :
return trackers + shared_trackers
else :
return trackers
def announceTracker ( self , tracker , * args , * * kwargs ) :
2019-07-03 04:48:07 +02:00
tracker_storage . setSiteAnnouncer ( self )
2018-08-26 02:45:37 +02:00
res = super ( SiteAnnouncerPlugin , self ) . announceTracker ( tracker , * args , * * kwargs )
if res :
latency = res
tracker_storage . onTrackerSuccess ( tracker , latency )
2018-08-26 22:52:21 +02:00
elif res is False :
2018-08-26 02:45:37 +02:00
tracker_storage . onTrackerError ( tracker )
return res
@PluginManager.registerTo ( " FileRequest " )
class FileRequestPlugin ( object ) :
def actionGetTrackers ( self , params ) :
2019-03-15 21:06:59 +01:00
shared_trackers = list ( tracker_storage . getWorkingTrackers ( " shared " ) . keys ( ) )
2019-07-03 19:24:50 +02:00
random . shuffle ( shared_trackers )
2018-08-26 02:45:37 +02:00
self . response ( { " trackers " : shared_trackers } )
@PluginManager.registerTo ( " FileServer " )
class FileServerPlugin ( object ) :
2019-01-20 19:12:11 +01:00
def portCheck ( self , * args , * * kwargs ) :
res = super ( FileServerPlugin , self ) . portCheck ( * args , * * kwargs )
2018-08-26 02:45:37 +02:00
if res and not config . tor == " always " and " Bootstrapper " in PluginManager . plugin_manager . plugin_names :
2019-01-26 20:42:27 +01:00
for ip in self . ip_external_list :
2019-01-24 15:20:08 +01:00
my_tracker_address = " zero:// %s : %s " % ( ip , config . fileserver_port )
tracker_storage . onTrackerFound ( my_tracker_address , my = True )
2018-08-26 02:45:37 +02:00
return res
2018-09-02 02:17:42 +02:00
2019-01-20 19:12:11 +01:00
2018-09-02 02:17:42 +02:00
@PluginManager.registerTo ( " ConfigPlugin " )
class ConfigPlugin ( object ) :
def createArguments ( self ) :
group = self . parser . add_argument_group ( " AnnounceShare plugin " )
2019-07-03 13:27:53 +02:00
group . add_argument ( ' --working_shared_trackers_limit ' , help = ' Stop discovering new shared trackers after this number of shared trackers reached (total) ' , default = 10 , type = int , metavar = ' limit ' )
2019-07-04 18:59:59 +02:00
group . add_argument ( ' --working_shared_trackers_limit_per_protocol ' , help = ' Stop discovering new shared trackers after this number of shared trackers reached per each supported protocol ' , default = " zero=5,other=2 " , metavar = ' limit ' )
2018-09-02 02:17:42 +02:00
return super ( ConfigPlugin , self ) . createArguments ( )