2015-01-12 02:03:45 +01:00
import os , json , logging , hashlib , re , time , string , random
from lib . subtl . subtl import UdpTrackerClient
import gevent
import util
from Config import config
from Peer import Peer
from Worker import WorkerManager
from Crypt import CryptHash
2015-01-17 18:50:56 +01:00
from Debug import Debug
2015-01-12 02:03:45 +01:00
import SiteManager
class Site :
def __init__ ( self , address , allow_create = True ) :
self . address = re . sub ( " [^A-Za-z0-9] " , " " , address ) # Make sure its correct address
self . address_short = " %s .. %s " % ( self . address [ : 6 ] , self . address [ - 4 : ] ) # Short address for logging
self . directory = " data/ %s " % self . address # Site data diretory
self . log = logging . getLogger ( " Site: %s " % self . address_short )
if not os . path . isdir ( self . directory ) :
if allow_create :
os . mkdir ( self . directory ) # Create directory if not found
else :
raise Exception ( " Directory not exists: %s " % self . directory )
self . content = None # Load content.json
self . peers = { } # Key: ip:port, Value: Peer.Peer
self . peer_blacklist = SiteManager . peer_blacklist # Ignore this peers (eg. myself)
self . last_announce = 0 # Last announce time to tracker
self . worker_manager = WorkerManager ( self ) # Handle site download from other peers
2015-01-18 22:52:19 +01:00
self . bad_files = { } # SHA512 check failed files, need to redownload
2015-01-12 02:03:45 +01:00
self . content_updated = None # Content.js update time
self . last_downloads = [ ] # Files downloaded in run of self.download()
self . notifications = [ ] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self . page_requested = False # Page viewed in browser
self . loadContent ( init = True ) # Load content.json
self . loadSettings ( ) # Load settings from sites.json
2015-01-18 22:52:19 +01:00
if not self . settings . get ( " auth_key " ) : # To auth user in site
self . settings [ " auth_key " ] = ' ' . join ( random . choice ( string . ascii_uppercase + string . ascii_lowercase + string . digits ) for _ in range ( 24 ) )
2015-01-12 02:03:45 +01:00
self . log . debug ( " New auth key: %s " % self . settings [ " auth_key " ] )
self . saveSettings ( )
2015-01-18 22:52:19 +01:00
if not self . settings . get ( " wrapper_key " ) : # To auth websocket permissions
self . settings [ " wrapper_key " ] = ' ' . join ( random . choice ( string . ascii_uppercase + string . ascii_lowercase + string . digits ) for _ in range ( 12 ) )
self . log . debug ( " New wrapper key: %s " % self . settings [ " wrapper_key " ] )
self . saveSettings ( )
2015-01-12 02:03:45 +01:00
self . websockets = [ ] # Active site websocket connections
# Add event listeners
self . addEventListeners ( )
# Load content.json to self.content
def loadContent ( self , init = False ) :
old_content = self . content
content_path = " %s /content.json " % self . directory
if os . path . isfile ( content_path ) :
try :
new_content = json . load ( open ( content_path ) )
except Exception , err :
2015-01-17 18:50:56 +01:00
self . log . error ( " Content.json load error: %s " % Debug . formatException ( err ) )
2015-01-12 02:03:45 +01:00
return None
else :
return None # Content.json not exits
try :
changed = [ ]
for inner_path , details in new_content [ " files " ] . items ( ) :
new_sha1 = details [ " sha1 " ]
if old_content and old_content [ " files " ] . get ( inner_path ) :
old_sha1 = old_content [ " files " ] [ inner_path ] [ " sha1 " ]
else :
old_sha1 = None
if old_sha1 != new_sha1 : changed . append ( inner_path )
self . content = new_content
except Exception , err :
2015-01-17 18:50:56 +01:00
self . log . error ( " Content.json parse error: %s " % Debug . formatException ( err ) )
2015-01-12 02:03:45 +01:00
return None # Content.json parse error
# Add to bad files
if not init :
for inner_path in changed :
self . bad_files [ inner_path ] = True
return changed
# Load site settings from data/sites.json
def loadSettings ( self ) :
sites_settings = json . load ( open ( " data/sites.json " ) )
if self . address in sites_settings :
self . settings = sites_settings [ self . address ]
else :
if self . address == config . homepage : # Add admin permissions to homepage
permissions = [ " ADMIN " ]
else :
permissions = [ ]
self . settings = { " own " : False , " serving " : True , " permissions " : permissions } # Default
return
# Save site settings to data/sites.json
def saveSettings ( self ) :
sites_settings = json . load ( open ( " data/sites.json " ) )
sites_settings [ self . address ] = self . settings
open ( " data/sites.json " , " w " ) . write ( json . dumps ( sites_settings , indent = 4 , sort_keys = True ) )
return
# Sercurity check and return path of site's file
def getPath ( self , inner_path ) :
inner_path = inner_path . replace ( " \\ " , " / " ) # Windows separator fix
inner_path = re . sub ( " ^ %s / " % re . escape ( self . directory ) , " " , inner_path ) # Remove site directory if begins with it
file_path = self . directory + " / " + inner_path
allowed_dir = os . path . abspath ( self . directory ) # Only files within this directory allowed
if " .. " in file_path or not os . path . dirname ( os . path . abspath ( file_path ) ) . startswith ( allowed_dir ) :
raise Exception ( " File not allowed: %s " % file_path )
return file_path
# Start downloading site
@util.Noparallel ( blocking = False )
def download ( self ) :
2015-01-17 18:50:56 +01:00
self . log . debug ( " Start downloading... %s " % self . bad_files )
2015-01-12 02:03:45 +01:00
self . announce ( )
found = self . needFile ( " content.json " , update = self . bad_files . get ( " content.json " ) )
if not found : return False # Could not download content.json
self . loadContent ( ) # Load the content.json
self . log . debug ( " Got content.json " )
evts = [ ]
self . last_downloads = [ " content.json " ] # Files downloaded in this run
for inner_path in self . content [ " files " ] . keys ( ) :
res = self . needFile ( inner_path , blocking = False , update = self . bad_files . get ( inner_path ) ) # No waiting for finish, return the event
if res != True : # Need downloading
self . last_downloads . append ( inner_path )
evts . append ( res ) # Append evt
self . log . debug ( " Downloading %s files... " % len ( evts ) )
s = time . time ( )
gevent . joinall ( evts )
self . log . debug ( " All file downloaded in %.2f s " % ( time . time ( ) - s ) )
# Update content.json from peers and download changed files
@util.Noparallel ( )
def update ( self ) :
self . loadContent ( ) # Reload content.json
self . content_updated = None
self . needFile ( " content.json " , update = True )
changed_files = self . loadContent ( )
if changed_files :
for changed_file in changed_files :
self . bad_files [ changed_file ] = True
self . checkFiles ( quick_check = True ) # Quick check files based on file size
if self . bad_files :
self . download ( )
return changed_files
# Update content.json on peers
def publish ( self , limit = 3 ) :
2015-01-15 23:24:51 +01:00
self . log . info ( " Publishing to %s / %s peers... " % ( limit , len ( self . peers ) ) )
2015-01-12 02:03:45 +01:00
published = 0
for key , peer in self . peers . items ( ) : # Send update command to each peer
result = { " exception " : " Timeout " }
try :
2015-01-15 23:24:51 +01:00
with gevent . Timeout ( 1 , False ) : # 1 sec timeout
2015-01-12 02:03:45 +01:00
result = peer . sendCmd ( " update " , {
" site " : self . address ,
" inner_path " : " content.json " ,
" body " : open ( self . getPath ( " content.json " ) ) . read ( ) ,
" peer " : ( config . ip_external , config . fileserver_port )
} )
except Exception , err :
2015-01-17 18:50:56 +01:00
result = { " exception " : Debug . formatException ( err ) }
2015-01-12 02:03:45 +01:00
if result and " ok " in result :
published + = 1
self . log . info ( " [OK] %s : %s " % ( key , result [ " ok " ] ) )
else :
self . log . info ( " [ERROR] %s : %s " % ( key , result ) )
if published > = limit : break
self . log . info ( " Successfuly published to %s peers " % published )
return published
# Check and download if file not exits
2015-01-14 02:41:13 +01:00
def needFile ( self , inner_path , update = False , blocking = True , peer = None , priority = 0 ) :
2015-01-12 02:03:45 +01:00
if os . path . isfile ( self . getPath ( inner_path ) ) and not update : # File exits, no need to do anything
return True
elif self . settings [ " serving " ] == False : # Site not serving
return False
else : # Wait until file downloaded
if not self . content : # No content.json, download it first!
self . log . debug ( " Need content.json first " )
self . announce ( )
if inner_path != " content.json " : # Prevent double download
2015-01-14 22:57:43 +01:00
task = self . worker_manager . addTask ( " content.json " , peer )
2015-01-12 02:03:45 +01:00
task . get ( )
self . loadContent ( )
if not self . content : return False
2015-01-14 02:41:13 +01:00
task = self . worker_manager . addTask ( inner_path , peer , priority = priority )
2015-01-12 02:03:45 +01:00
if blocking :
return task . get ( )
else :
return task
# Add or update a peer to site
def addPeer ( self , ip , port , return_peer = False ) :
key = " %s : %s " % ( ip , port )
if key in self . peers : # Already has this ip
self . peers [ key ] . found ( )
if return_peer : # Always return peer
return self . peers [ key ]
else :
return False
else : # New peer
2015-01-13 21:19:40 +01:00
peer = Peer ( ip , port , self )
2015-01-12 02:03:45 +01:00
self . peers [ key ] = peer
return peer
# Add myself and get other peers from tracker
def announce ( self , force = False ) :
if time . time ( ) < self . last_announce + 15 and not force : return # No reannouncing within 15 secs
self . last_announce = time . time ( )
for protocol , ip , port in SiteManager . TRACKERS :
if protocol == " udp " :
2015-01-18 22:52:19 +01:00
self . log . debug ( " Announcing to %s :// %s : %s ... " % ( protocol , ip , port ) )
2015-01-12 02:03:45 +01:00
tracker = UdpTrackerClient ( ip , port )
tracker . peer_port = config . fileserver_port
try :
tracker . connect ( )
tracker . poll_once ( )
2015-01-15 23:24:51 +01:00
tracker . announce ( info_hash = hashlib . sha1 ( self . address ) . hexdigest ( ) , num_want = 50 )
2015-01-12 02:03:45 +01:00
back = tracker . poll_once ( )
peers = back [ " response " ] [ " peers " ]
2015-01-17 18:50:56 +01:00
except Exception , err :
self . log . error ( " Tracker error: %s " % Debug . formatException ( err ) )
2015-01-12 02:03:45 +01:00
time . sleep ( 1 )
2015-01-17 18:50:56 +01:00
continue
added = 0
for peer in peers :
if ( peer [ " addr " ] , peer [ " port " ] ) in self . peer_blacklist : # Ignore blacklist (eg. myself)
continue
if self . addPeer ( peer [ " addr " ] , peer [ " port " ] ) : added + = 1
if added :
self . worker_manager . onPeers ( )
self . updateWebsocket ( peers_added = added )
self . log . debug ( " Found %s peers, new: %s " % ( len ( peers ) , added ) )
break # Successful announcing, break the list
2015-01-12 02:03:45 +01:00
else :
pass # TODO: http tracker support
# Check and try to fix site files integrity
def checkFiles ( self , quick_check = True ) :
self . log . debug ( " Checking files... Quick: %s " % quick_check )
bad_files = self . verifyFiles ( quick_check )
if bad_files :
for bad_file in bad_files :
self . bad_files [ bad_file ] = True
# - Events -
# Add event listeners
def addEventListeners ( self ) :
self . onFileStart = util . Event ( ) # If WorkerManager added new task
self . onFileDone = util . Event ( ) # If WorkerManager successfuly downloaded a file
self . onFileFail = util . Event ( ) # If WorkerManager failed to download a file
self . onComplete = util . Event ( ) # All file finished
self . onFileStart . append ( lambda inner_path : self . fileStarted ( ) ) # No parameters to make Noparallel batching working
self . onFileDone . append ( lambda inner_path : self . fileDone ( inner_path ) )
self . onFileFail . append ( lambda inner_path : self . fileFailed ( inner_path ) )
# Send site status update to websocket clients
def updateWebsocket ( self , * * kwargs ) :
if kwargs :
param = { " event " : kwargs . items ( ) [ 0 ] }
else :
param = None
for ws in self . websockets :
ws . event ( " siteChanged " , self , param )
# File download started
@util.Noparallel ( blocking = False )
def fileStarted ( self ) :
time . sleep ( 0.001 ) # Wait for other files adds
self . updateWebsocket ( file_started = True )
# File downloaded successful
def fileDone ( self , inner_path ) :
# File downloaded, remove it from bad files
if inner_path in self . bad_files :
2015-01-12 19:11:35 +01:00
self . log . info ( " Bad file solved: %s " % inner_path )
2015-01-12 02:03:45 +01:00
del ( self . bad_files [ inner_path ] )
# Update content.json last downlad time
if inner_path == " content.json " :
self . content_updated = time . time ( )
self . updateWebsocket ( file_done = inner_path )
# File download failed
def fileFailed ( self , inner_path ) :
if inner_path == " content.json " :
self . content_updated = False
self . log . error ( " Can ' t update content.json " )
self . updateWebsocket ( file_failed = inner_path )
# - Sign and verify -
# Verify fileobj using sha1 in content.json
def verifyFile ( self , inner_path , file , force = False ) :
if inner_path == " content.json " : # Check using sign
from Crypt import CryptBitcoin
try :
content = json . load ( file )
if self . content and not force :
if self . content [ " modified " ] == content [ " modified " ] : # Ignore, have the same content.json
return None
elif self . content [ " modified " ] > content [ " modified " ] : # We have newer
2015-01-14 02:41:13 +01:00
self . log . debug ( " We have newer content.json (Our: %s , Sent: %s ) " % ( self . content [ " modified " ] , content [ " modified " ] ) )
2015-01-12 02:03:45 +01:00
return False
if content [ " modified " ] > time . time ( ) + 60 * 60 * 24 : # Content modified in the far future (allow 1 day window)
self . log . error ( " Content.json modify is in the future! " )
return False
# Check sign
sign = content [ " sign " ]
del ( content [ " sign " ] ) # The file signed without the sign
sign_content = json . dumps ( content , sort_keys = True ) # Dump the json to string to remove whitepsace
return CryptBitcoin . verify ( sign_content , self . address , sign )
except Exception , err :
2015-01-17 18:50:56 +01:00
self . log . error ( " Verify sign error: %s " % Debug . formatException ( err ) )
2015-01-12 02:03:45 +01:00
return False
else : # Check using sha1 hash
if self . content and inner_path in self . content [ " files " ] :
2015-01-18 22:52:19 +01:00
if " sha512 " in self . content [ " files " ] [ inner_path ] : # Use sha512 to verify if possible
return CryptHash . sha512sum ( file ) == self . content [ " files " ] [ inner_path ] [ " sha512 " ]
else : # Backward compatiblity
return CryptHash . sha1sum ( file ) == self . content [ " files " ] [ inner_path ] [ " sha1 " ]
2015-01-12 02:03:45 +01:00
else : # File not in content.json
self . log . error ( " File not in content.json: %s " % inner_path )
return False
2015-01-18 22:52:19 +01:00
# Verify all files sha512sum using content.json
2015-01-12 02:03:45 +01:00
def verifyFiles ( self , quick_check = False ) : # Fast = using file size
bad_files = [ ]
if not self . content : # No content.json, download it first
self . needFile ( " content.json " , update = True ) # Force update to fix corrupt file
self . loadContent ( ) # Reload content.json
for inner_path in self . content [ " files " ] . keys ( ) :
file_path = self . getPath ( inner_path )
if not os . path . isfile ( file_path ) :
self . log . error ( " [MISSING] %s " % inner_path )
bad_files . append ( inner_path )
continue
if quick_check :
ok = os . path . getsize ( file_path ) == self . content [ " files " ] [ inner_path ] [ " size " ]
else :
ok = self . verifyFile ( inner_path , open ( file_path , " rb " ) )
if ok :
self . log . debug ( " [OK] %s " % inner_path )
else :
self . log . error ( " [ERROR] %s " % inner_path )
bad_files . append ( inner_path )
return bad_files
# Create and sign content.json using private key
def signContent ( self , privatekey = None ) :
if not self . content : # New site
self . log . info ( " Site not exits yet, loading default content.json values... " )
2015-01-16 11:52:42 +01:00
self . content = { " files " : { } , " title " : " %s - ZeroNet_ " % self . address , " sign " : " " , " modified " : 0.0 , " description " : " " , " address " : self . address , " ignore " : " " , " zeronet_version " : config . version } # Default content.json
2015-01-12 02:03:45 +01:00
self . log . info ( " Opening site data directory: %s ... " % self . directory )
hashed_files = { }
for root , dirs , files in os . walk ( self . directory ) :
for file_name in files :
file_path = self . getPath ( " %s / %s " % ( root , file_name ) )
if file_name == " content.json " or ( self . content [ " ignore " ] and re . match ( self . content [ " ignore " ] , file_path . replace ( self . directory + " / " , " " ) ) ) : # Dont add content.json and ignore regexp pattern definied in content.json
self . log . info ( " - [SKIPPED] %s " % file_path )
else :
2015-01-18 22:52:19 +01:00
sha1sum = CryptHash . sha1sum ( file_path ) # Calculate sha1 sum of file
sha512sum = CryptHash . sha512sum ( file_path ) # Calculate sha512 sum of file
2015-01-12 02:03:45 +01:00
inner_path = re . sub ( " ^ %s / " % re . escape ( self . directory ) , " " , file_path )
2015-01-18 22:52:19 +01:00
self . log . info ( " - %s (SHA512: %s ) " % ( file_path , sha512sum ) )
hashed_files [ inner_path ] = { " sha1 " : sha1sum , " sha512 " : sha512sum , " size " : os . path . getsize ( file_path ) }
2015-01-12 02:03:45 +01:00
# Generate new content.json
2015-01-18 22:52:19 +01:00
self . log . info ( " Adding timestamp and sha512sums to new content.json... " )
2015-01-12 02:03:45 +01:00
content = self . content . copy ( ) # Create a copy of current content.json
2015-01-18 22:52:19 +01:00
content [ " address " ] = self . address
content [ " files " ] = hashed_files # Add files sha512 hash
2015-01-13 21:19:40 +01:00
content [ " modified " ] = time . time ( ) # Add timestamp
2015-01-16 11:52:42 +01:00
content [ " zeronet_version " ] = config . version # Signer's zeronet version
del ( content [ " sign " ] ) # Delete old sign
2015-01-12 02:03:45 +01:00
# Signing content
from Crypt import CryptBitcoin
self . log . info ( " Verifying private key... " )
privatekey_address = CryptBitcoin . privatekeyToAddress ( privatekey )
if self . address != privatekey_address :
return self . log . error ( " Private key invalid! Site address: %s , Private key address: %s " % ( self . address , privatekey_address ) )
self . log . info ( " Signing modified content.json... " )
sign_content = json . dumps ( content , sort_keys = True )
self . log . debug ( " Content: %s " % sign_content )
sign = CryptBitcoin . sign ( sign_content , privatekey )
content [ " sign " ] = sign
# Saving modified content.json
self . log . info ( " Saving to %s /content.json... " % self . directory )
open ( " %s /content.json " % self . directory , " w " ) . write ( json . dumps ( content , indent = 4 , sort_keys = True ) )
self . log . info ( " Site signed! " )