2015-04-12 23:59:22 +02:00
import os , json , logging , hashlib , re , time , string , random , sys , binascii , struct , socket , urllib , urllib2
2015-01-12 02:03:45 +01:00
from lib . subtl . subtl import UdpTrackerClient
2015-04-12 23:59:22 +02:00
from lib import bencode
2015-01-12 02:03:45 +01:00
import gevent
import util
from Config import config
from Peer import Peer
from Worker import WorkerManager
from Crypt import CryptHash
2015-01-17 18:50:56 +01:00
from Debug import Debug
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
from Content import ContentManager
2015-03-19 21:19:14 +01:00
from SiteStorage import SiteStorage
2015-01-12 02:03:45 +01:00
import SiteManager
class Site :
def __init__ ( self , address , allow_create = True ) :
self . address = re . sub ( " [^A-Za-z0-9] " , " " , address ) # Make sure its correct address
self . address_short = " %s .. %s " % ( self . address [ : 6 ] , self . address [ - 4 : ] ) # Short address for logging
self . log = logging . getLogger ( " Site: %s " % self . address_short )
self . content = None # Load content.json
self . peers = { } # Key: ip:port, Value: Peer.Peer
self . peer_blacklist = SiteManager . peer_blacklist # Ignore this peers (eg. myself)
self . last_announce = 0 # Last announce time to tracker
self . worker_manager = WorkerManager ( self ) # Handle site download from other peers
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
self . bad_files = { } # SHA512 check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
2015-01-12 02:03:45 +01:00
self . content_updated = None # Content.js update time
self . notifications = [ ] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self . page_requested = False # Page viewed in browser
2015-03-19 21:19:14 +01:00
self . storage = SiteStorage ( self , allow_create = allow_create ) # Save and load site files
2015-01-12 02:03:45 +01:00
self . loadSettings ( ) # Load settings from sites.json
2015-02-14 14:05:00 +01:00
self . content_manager = ContentManager ( self ) # Load contents
2015-01-12 02:03:45 +01:00
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
if not self . settings . get ( " auth_key " ) : # To auth user in site (Obsolete, will be removed)
2015-01-18 22:52:19 +01:00
self . settings [ " auth_key " ] = ' ' . join ( random . choice ( string . ascii_uppercase + string . ascii_lowercase + string . digits ) for _ in range ( 24 ) )
2015-01-12 02:03:45 +01:00
self . log . debug ( " New auth key: %s " % self . settings [ " auth_key " ] )
self . saveSettings ( )
2015-01-18 22:52:19 +01:00
if not self . settings . get ( " wrapper_key " ) : # To auth websocket permissions
self . settings [ " wrapper_key " ] = ' ' . join ( random . choice ( string . ascii_uppercase + string . ascii_lowercase + string . digits ) for _ in range ( 12 ) )
self . log . debug ( " New wrapper key: %s " % self . settings [ " wrapper_key " ] )
self . saveSettings ( )
2015-01-12 02:03:45 +01:00
self . websockets = [ ] # Active site websocket connections
# Add event listeners
self . addEventListeners ( )
2015-03-11 01:12:53 +01:00
def __str__ ( self ) :
return " Site %s " % self . address_short
def __repr__ ( self ) :
return " < %s > " % self . __str__ ( )
2015-01-12 02:03:45 +01:00
# Load site settings from data/sites.json
def loadSettings ( self ) :
sites_settings = json . load ( open ( " data/sites.json " ) )
if self . address in sites_settings :
self . settings = sites_settings [ self . address ]
else :
if self . address == config . homepage : # Add admin permissions to homepage
permissions = [ " ADMIN " ]
else :
permissions = [ ]
self . settings = { " own " : False , " serving " : True , " permissions " : permissions } # Default
return
# Save site settings to data/sites.json
def saveSettings ( self ) :
sites_settings = json . load ( open ( " data/sites.json " ) )
sites_settings [ self . address ] = self . settings
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
open ( " data/sites.json " , " w " ) . write ( json . dumps ( sites_settings , indent = 2 , sort_keys = True ) )
2015-01-12 02:03:45 +01:00
return
2015-02-14 14:05:00 +01:00
# Max site size in MB
def getSizeLimit ( self ) :
return self . settings . get ( " size_limit " , config . size_limit )
# Next size limit based on current size
def getNextSizeLimit ( self ) :
size_limits = [ 10 , 20 , 50 , 100 , 200 , 500 , 1000 , 2000 , 5000 , 10000 , 20000 , 50000 , 100000 ]
size = self . settings . get ( " size " , 0 )
for size_limit in size_limits :
if size * 1.2 < size_limit * 1024 * 1024 :
return size_limit
return 999999
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
# Download all file from content.json
@util.Noparallel ( blocking = True )
def downloadContent ( self , inner_path , download_files = True , peer = None ) :
s = time . time ( )
self . log . debug ( " Downloading %s ... " % inner_path )
found = self . needFile ( inner_path , update = self . bad_files . get ( inner_path ) )
content_inner_dir = self . content_manager . toDir ( inner_path )
if not found : return False # Could not download content.json
self . log . debug ( " Got %s " % inner_path )
changed = self . content_manager . loadContent ( inner_path , load_includes = False )
# Start download files
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
file_threads = [ ]
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
if download_files :
for file_relative_path in self . content_manager . contents [ inner_path ] . get ( " files " , { } ) . keys ( ) :
file_inner_path = content_inner_dir + file_relative_path
res = self . needFile ( file_inner_path , blocking = False , update = self . bad_files . get ( file_inner_path ) , peer = peer ) # No waiting for finish, return the event
if res != True : # Need downloading
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
file_threads . append ( res ) # Append evt
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
# Wait for includes download
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
include_threads = [ ]
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
for file_relative_path in self . content_manager . contents [ inner_path ] . get ( " includes " , { } ) . keys ( ) :
file_inner_path = content_inner_dir + file_relative_path
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
include_thread = gevent . spawn ( self . downloadContent , file_inner_path , download_files = download_files , peer = peer )
include_threads . append ( include_thread )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
self . log . debug ( " %s : Downloading %s includes... " % ( inner_path , len ( include_threads ) ) )
gevent . joinall ( include_threads )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
self . log . debug ( " %s : Includes downloaded " % inner_path )
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
2015-02-26 01:32:27 +01:00
self . log . debug ( " %s : Downloading %s files, changed: %s ... " % ( inner_path , len ( file_threads ) , len ( changed ) ) )
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
gevent . joinall ( file_threads )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
self . log . debug ( " %s : All file downloaded in %.2f s " % ( inner_path , time . time ( ) - s ) )
return True
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
# Return bad files with less than 3 retry
def getReachableBadFiles ( self ) :
if not self . bad_files : return False
return [ bad_file for bad_file , retry in self . bad_files . iteritems ( ) if retry < 3 ]
2015-04-08 01:57:55 +02:00
# Retry download bad files
def retryBadFiles ( self ) :
for bad_file in self . bad_files . keys ( ) :
self . needFile ( bad_file , update = True , blocking = False )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
# Download all files of the site
2015-01-12 02:03:45 +01:00
@util.Noparallel ( blocking = False )
2015-02-14 14:05:00 +01:00
def download ( self , check_size = False ) :
2015-01-17 18:50:56 +01:00
self . log . debug ( " Start downloading... %s " % self . bad_files )
2015-04-12 23:59:22 +02:00
gevent . spawn ( self . announce )
2015-02-14 14:05:00 +01:00
if check_size : # Check the size first
valid = downloadContent ( download_files = False )
if not valid : return False # Cant download content.jsons or size is not fits
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
found = self . downloadContent ( " content.json " )
return found
2015-01-12 02:03:45 +01:00
# Update content.json from peers and download changed files
@util.Noparallel ( )
def update ( self ) :
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
self . content_manager . loadContent ( " content.json " ) # Reload content.json
2015-01-12 02:03:45 +01:00
self . content_updated = None
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
# Download all content.json again
2015-02-26 01:32:27 +01:00
content_threads = [ ]
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
for inner_path in self . content_manager . contents . keys ( ) :
2015-02-26 01:32:27 +01:00
content_threads . append ( self . needFile ( inner_path , update = True , blocking = False ) )
self . log . debug ( " Waiting %s content.json to finish... " % len ( content_threads ) )
gevent . joinall ( content_threads )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
changed = self . content_manager . loadContent ( " content.json " )
if changed :
for changed_file in changed :
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
self . bad_files [ changed_file ] = self . bad_files . get ( changed_file , 0 ) + 1
2015-03-19 21:19:14 +01:00
if not self . settings [ " own " ] : self . storage . checkFiles ( quick_check = True ) # Quick check files based on file size
2015-04-17 00:34:08 +02:00
2015-01-12 02:03:45 +01:00
if self . bad_files :
self . download ( )
2015-02-14 14:05:00 +01:00
self . settings [ " size " ] = self . content_manager . getTotalSize ( ) # Update site size
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
return changed
2015-01-12 02:03:45 +01:00
2015-03-19 21:19:14 +01:00
# Publish worker
2015-04-08 01:57:55 +02:00
def publisher ( self , inner_path , peers , published , limit , event_done = None ) :
2015-04-17 00:34:08 +02:00
file_size = self . storage . getSize ( inner_path )
body = self . storage . read ( inner_path )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
while 1 :
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
if not peers or len ( published ) > = limit :
2015-04-08 01:57:55 +02:00
if event_done : event_done . set ( True )
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
break # All peers done, or published engouht
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
peer = peers . pop ( 0 )
2015-04-17 00:34:08 +02:00
if peer . connection and peer . connection . last_ping_delay : # Peer connected
timeout = timeout = 5 + int ( file_size / 1024 ) + peer . connection . last_ping_delay # Timeout: 5sec + size in kb + last_ping
else :
timeout = timeout = 5 + int ( file_size / 1024 ) # Timeout: 5sec + size in kb
2015-01-12 02:03:45 +01:00
result = { " exception " : " Timeout " }
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
for retry in range ( 2 ) :
try :
with gevent . Timeout ( timeout , False ) :
result = peer . request ( " update " , {
" site " : self . address ,
" inner_path " : inner_path ,
2015-04-17 00:34:08 +02:00
" body " : body ,
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
" peer " : ( config . ip_external , config . fileserver_port )
} )
if result : break
except Exception , err :
result = { " exception " : Debug . formatException ( err ) }
2015-01-12 02:03:45 +01:00
if result and " ok " in result :
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
published . append ( peer )
self . log . info ( " [OK] %s : %s " % ( peer . key , result [ " ok " ] ) )
2015-01-12 02:03:45 +01:00
else :
2015-04-15 23:33:21 +02:00
if result == { " exception " : " Timeout " } : peer . onConnectionError ( )
2015-04-08 01:57:55 +02:00
self . log . info ( " [FAILED] %s : %s " % ( peer . key , result ) )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
# Update content.json on peers
2015-04-03 02:47:51 +02:00
def publish ( self , limit = 5 , inner_path = " content.json " ) :
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
self . log . info ( " Publishing to %s / %s peers... " % ( limit , len ( self . peers ) ) )
2015-04-17 00:34:08 +02:00
published = [ ] # Successfully published (Peer)
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
publishers = [ ] # Publisher threads
peers = self . peers . values ( )
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
random . shuffle ( peers )
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
event_done = gevent . event . AsyncResult ( )
2015-03-31 01:01:39 +02:00
for i in range ( min ( len ( self . peers ) , limit , 5 ) ) : # Max 5 thread
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
publisher = gevent . spawn ( self . publisher , inner_path , peers , published , limit , event_done )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
publishers . append ( publisher )
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
event_done . get ( ) # Wait for done
if len ( published ) < min ( len ( self . peers ) , limit ) : time . sleep ( 0.2 ) # If less than we need sleep a bit
2015-04-03 14:31:15 +02:00
if len ( published ) == 0 : gevent . joinall ( publishers ) # No successful publish, wait for all publisher
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
2015-04-17 00:34:08 +02:00
# Make sure the connected passive peers got the update
passive_peers = [ peer for peer in peers if peer . connection and not peer . connection . closed and peer . key . endswith ( " :0 " ) and peer not in published ] # Every connected passive peer that we not published to
for peer in passive_peers :
gevent . spawn ( self . publisher , inner_path , passive_peers , published , limit = 10 )
self . log . info ( " Successfuly published to %s peers, publishing to %s more passive peers " % ( len ( published ) , len ( passive_peers ) ) )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
return len ( published )
2015-01-12 02:03:45 +01:00
# Check and download if file not exits
2015-01-14 02:41:13 +01:00
def needFile ( self , inner_path , update = False , blocking = True , peer = None , priority = 0 ) :
2015-03-19 21:19:14 +01:00
if self . storage . isFile ( inner_path ) and not update : # File exits, no need to do anything
2015-01-12 02:03:45 +01:00
return True
elif self . settings [ " serving " ] == False : # Site not serving
return False
else : # Wait until file downloaded
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
self . bad_files [ inner_path ] = True # Mark as bad file
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
if not self . content_manager . contents . get ( " content.json " ) : # No content.json, download it first!
2015-01-12 02:03:45 +01:00
self . log . debug ( " Need content.json first " )
2015-04-12 23:59:22 +02:00
gevent . spawn ( self . announce )
2015-01-12 02:03:45 +01:00
if inner_path != " content.json " : # Prevent double download
2015-01-14 22:57:43 +01:00
task = self . worker_manager . addTask ( " content.json " , peer )
2015-01-12 02:03:45 +01:00
task . get ( )
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
self . content_manager . loadContent ( )
if not self . content_manager . contents . get ( " content.json " ) : return False # Content.json download failed
if not inner_path . endswith ( " content.json " ) and not self . content_manager . getFileInfo ( inner_path ) : # No info for file, download all content.json first
self . log . debug ( " No info for %s , waiting for all content.json " % inner_path )
success = self . downloadContent ( " content.json " , download_files = False )
if not success : return False
2015-04-12 23:59:22 +02:00
if not self . content_manager . getFileInfo ( inner_path ) : return False # Still no info for file
2015-01-12 02:03:45 +01:00
2015-01-14 02:41:13 +01:00
task = self . worker_manager . addTask ( inner_path , peer , priority = priority )
2015-01-12 02:03:45 +01:00
if blocking :
return task . get ( )
else :
return task
# Add or update a peer to site
def addPeer ( self , ip , port , return_peer = False ) :
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
if not ip : return False
2015-04-17 23:12:22 +02:00
if ( ip , port ) in self . peer_blacklist : return False # Ignore blacklist (eg. myself)
2015-01-12 02:03:45 +01:00
key = " %s : %s " % ( ip , port )
if key in self . peers : # Already has this ip
2015-04-13 23:08:57 +02:00
#self.peers[key].found()
2015-01-12 02:03:45 +01:00
if return_peer : # Always return peer
return self . peers [ key ]
else :
return False
else : # New peer
2015-01-13 21:19:40 +01:00
peer = Peer ( ip , port , self )
2015-01-12 02:03:45 +01:00
self . peers [ key ] = peer
return peer
2015-04-13 23:08:57 +02:00
# Gather peer from connected peers
@util.Noparallel ( blocking = False )
2015-04-15 23:33:21 +02:00
def announcePex ( self , query_num = 2 , need_num = 5 ) :
2015-04-13 23:08:57 +02:00
peers = [ peer for peer in self . peers . values ( ) if peer . connection and peer . connection . connected ] # Connected peers
if len ( peers ) == 0 : # Small number of connected peers for this site, connect to any
2015-04-17 23:12:22 +02:00
self . log . debug ( " Small number of peers detected...query all of peers using pex " )
2015-04-13 23:08:57 +02:00
peers = self . peers . values ( )
need_num = 10
random . shuffle ( peers )
done = 0
added = 0
for peer in peers :
2015-04-13 23:56:08 +02:00
if peer . connection : # Has connection
2015-04-17 23:12:22 +02:00
if " port_opened " in peer . connection . handshake : # This field added recently, so probably has has peer exchange
2015-04-13 23:56:08 +02:00
res = peer . pex ( need_num = need_num )
else :
res = False
else : # No connection
res = peer . pex ( need_num = need_num )
2015-04-17 23:12:22 +02:00
if type ( res ) == int : # We have result
2015-04-13 23:08:57 +02:00
done + = 1
added + = res
2015-04-17 23:12:22 +02:00
if res :
2015-04-13 23:08:57 +02:00
self . worker_manager . onPeers ( )
2015-04-17 23:12:22 +02:00
self . updateWebsocket ( peers_added = res )
2015-04-13 23:08:57 +02:00
if done == query_num : break
2015-04-17 23:12:22 +02:00
self . log . debug ( " Queried pex from %s peers got %s new peers. " % ( done , added ) )
2015-04-13 23:08:57 +02:00
2015-01-12 02:03:45 +01:00
# Add myself and get other peers from tracker
def announce ( self , force = False ) :
2015-04-12 23:59:22 +02:00
if time . time ( ) < self . last_announce + 60 and not force : return # No reannouncing within 60 secs
2015-01-12 02:03:45 +01:00
self . last_announce = time . time ( )
2015-02-10 00:08:25 +01:00
errors = [ ]
2015-04-12 23:59:22 +02:00
address_hash = hashlib . sha1 ( self . address ) . hexdigest ( )
my_peer_id = sys . modules [ " main " ] . file_server . peer_id
2015-04-15 02:54:10 +02:00
if sys . modules [ " main " ] . file_server . port_opened :
2015-04-12 23:59:22 +02:00
fileserver_port = config . fileserver_port
else : # Port not opened, report port 0
2015-04-15 02:54:10 +02:00
fileserver_port = 0
2015-04-12 23:59:22 +02:00
s = time . time ( )
announced = 0
2015-01-12 02:03:45 +01:00
for protocol , ip , port in SiteManager . TRACKERS :
2015-04-12 23:59:22 +02:00
if protocol == " udp " : # Udp tracker
if config . disable_udp : continue # No udp supported
2015-01-12 02:03:45 +01:00
tracker = UdpTrackerClient ( ip , port )
2015-04-12 23:59:22 +02:00
tracker . peer_port = fileserver_port
2015-01-12 02:03:45 +01:00
try :
tracker . connect ( )
tracker . poll_once ( )
2015-04-12 23:59:22 +02:00
tracker . announce ( info_hash = address_hash , num_want = 50 )
2015-01-12 02:03:45 +01:00
back = tracker . poll_once ( )
peers = back [ " response " ] [ " peers " ]
2015-01-17 18:50:56 +01:00
except Exception , err :
2015-02-10 00:08:25 +01:00
errors . append ( " %s :// %s : %s " % ( protocol , ip , port ) )
2015-01-17 18:50:56 +01:00
continue
2015-04-12 23:59:22 +02:00
else : # Http tracker
params = {
' info_hash ' : binascii . a2b_hex ( address_hash ) ,
' peer_id ' : my_peer_id , ' port ' : fileserver_port ,
' uploaded ' : 0 , ' downloaded ' : 0 , ' left ' : 0 , ' compact ' : 1 , ' numwant ' : 30 ,
' event ' : ' started '
}
try :
url = " http:// " + ip + " ? " + urllib . urlencode ( params )
# Load url
opener = urllib2 . build_opener ( )
response = opener . open ( url , timeout = 10 ) . read ( )
# Decode peers
peer_data = bencode . decode ( response ) [ " peers " ]
peer_count = len ( peer_data ) / 6
peers = [ ]
for peer_offset in xrange ( peer_count ) :
off = 6 * peer_offset
peer = peer_data [ off : off + 6 ]
addr , port = struct . unpack ( ' !LH ' , peer )
peers . append ( { " addr " : socket . inet_ntoa ( struct . pack ( ' !L ' , addr ) ) , " port " : port } )
except Exception , err :
self . log . debug ( " Http tracker %s error: %s " % ( url , err ) )
errors . append ( " %s :// %s " % ( protocol , ip ) )
continue
# Adding peers
added = 0
for peer in peers :
if not peer [ " port " ] : continue # Dont add peers with port 0
if self . addPeer ( peer [ " addr " ] , peer [ " port " ] ) : added + = 1
if added :
self . worker_manager . onPeers ( )
self . updateWebsocket ( peers_added = added )
self . log . debug ( " Found %s peers, new: %s " % ( len ( peers ) , added ) )
announced + = 1
2015-01-30 18:46:48 +01:00
2015-04-12 23:59:22 +02:00
# Save peers num
self . settings [ " peers " ] = len ( self . peers )
self . saveSettings ( )
2015-02-10 00:08:25 +01:00
if len ( errors ) < len ( SiteManager . TRACKERS ) : # Less errors than total tracker nums
2015-04-17 23:12:22 +02:00
self . log . debug ( " Announced port %s to %s trackers in %.3f s, errors: %s " % ( fileserver_port , announced , time . time ( ) - s , errors ) )
2015-01-30 18:46:48 +01:00
else :
2015-04-12 23:59:22 +02:00
self . log . error ( " Announced to %s trackers in %.3f s, failed " % ( announced , time . time ( ) - s ) )
2015-01-12 02:03:45 +01:00
2015-04-13 23:08:57 +02:00
if not [ peer for peer in self . peers . values ( ) if peer . connection and peer . connection . connected ] : # If no connected peer yet then wait for connections
gevent . spawn_later ( 3 , self . announcePex , need_num = 10 ) # Spawn 3 secs later
# self.onFileDone.once(lambda inner_path: self.announcePex(need_num=10), "announcePex_%s" % self.address) # After first file downloaded try to find more peers using pex
else : # Else announce immediately
self . announcePex ( )
2015-01-12 02:03:45 +01:00
2015-04-17 00:34:08 +02:00
# Keep connections to get the updates (required for passive clients)
2015-04-03 02:47:51 +02:00
def needConnections ( self ) :
2015-04-17 23:12:22 +02:00
need = min ( len ( self . peers ) , 3 ) # Need 3 peer, but max total peers
2015-04-03 02:47:51 +02:00
connected = 0
for peer in self . peers . values ( ) : # Check current connected number
if peer . connection and peer . connection . connected :
connected + = 1
self . log . debug ( " Need connections: %s , Current: %s , Total: %s " % ( need , connected , len ( self . peers ) ) )
if connected < need : # Need more than we have
for peer in self . peers . values ( ) :
if not peer . connection or not peer . connection . connected : # No peer connection or disconnected
2015-04-14 02:37:31 +02:00
peer . pex ( ) # Initiate peer exchange
2015-04-03 02:47:51 +02:00
if peer . connection and peer . connection . connected : connected + = 1 # Successfully connected
if connected > = need : break
return connected
2015-01-21 12:58:26 +01:00
2015-04-15 23:33:21 +02:00
# Return: Probably working, connectable Peers
def getConnectablePeers ( self , need_num = 5 , ignore = [ ] ) :
peers = self . peers . values ( )
random . shuffle ( peers )
found = [ ]
for peer in peers :
if peer . key . endswith ( " :0 " ) : continue # Not connectable
if not peer . connection : continue # No connection
if peer . key in ignore : continue # The requester has this peer
if time . time ( ) - peer . connection . last_recv_time > 60 * 60 * 2 : # Last message more than 2 hours ago
peer . connection = None # Cleanup: Dead connection
continue
found . append ( peer )
if len ( found ) > = need_num : break # Found requested number of peers
if not found and not ignore : # Not found any peer and the requester dont have any, return not that good peers
found = [ peer for peer in peers if not peer . key . endswith ( " :0 " ) and peer . key not in ignore ] [ 0 : need_num ]
return found
2015-01-21 12:58:26 +01:00
2015-01-12 02:03:45 +01:00
# - Events -
# Add event listeners
def addEventListeners ( self ) :
self . onFileStart = util . Event ( ) # If WorkerManager added new task
self . onFileDone = util . Event ( ) # If WorkerManager successfuly downloaded a file
self . onFileFail = util . Event ( ) # If WorkerManager failed to download a file
self . onComplete = util . Event ( ) # All file finished
self . onFileStart . append ( lambda inner_path : self . fileStarted ( ) ) # No parameters to make Noparallel batching working
self . onFileDone . append ( lambda inner_path : self . fileDone ( inner_path ) )
self . onFileFail . append ( lambda inner_path : self . fileFailed ( inner_path ) )
# Send site status update to websocket clients
def updateWebsocket ( self , * * kwargs ) :
if kwargs :
param = { " event " : kwargs . items ( ) [ 0 ] }
else :
param = None
for ws in self . websockets :
ws . event ( " siteChanged " , self , param )
# File download started
@util.Noparallel ( blocking = False )
def fileStarted ( self ) :
time . sleep ( 0.001 ) # Wait for other files adds
self . updateWebsocket ( file_started = True )
# File downloaded successful
def fileDone ( self , inner_path ) :
# File downloaded, remove it from bad files
if inner_path in self . bad_files :
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
self . log . debug ( " Bad file solved: %s " % inner_path )
2015-01-12 02:03:45 +01:00
del ( self . bad_files [ inner_path ] )
# Update content.json last downlad time
if inner_path == " content.json " :
self . content_updated = time . time ( )
self . updateWebsocket ( file_done = inner_path )
# File download failed
def fileFailed ( self , inner_path ) :
if inner_path == " content.json " :
self . content_updated = False
2015-04-15 23:33:21 +02:00
self . log . debug ( " Can ' t update content.json " )
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
if inner_path in self . bad_files :
self . bad_files [ inner_path ] = self . bad_files . get ( inner_path , 0 ) + 1
2015-01-12 02:03:45 +01:00
self . updateWebsocket ( file_failed = inner_path )