2015-01-12 02:03:45 +01:00
|
|
|
import os, json, logging, hashlib, re, time, string, random
|
|
|
|
from lib.subtl.subtl import UdpTrackerClient
|
|
|
|
import gevent
|
|
|
|
import util
|
|
|
|
from Config import config
|
|
|
|
from Peer import Peer
|
|
|
|
from Worker import WorkerManager
|
|
|
|
from Crypt import CryptHash
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
from Content import ContentManager
|
2015-01-12 02:03:45 +01:00
|
|
|
import SiteManager
|
|
|
|
|
|
|
|
class Site:
|
|
|
|
def __init__(self, address, allow_create=True):
|
|
|
|
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
|
|
|
|
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
|
|
|
|
self.directory = "data/%s" % self.address # Site data diretory
|
|
|
|
self.log = logging.getLogger("Site:%s" % self.address_short)
|
|
|
|
|
|
|
|
if not os.path.isdir(self.directory):
|
|
|
|
if allow_create:
|
|
|
|
os.mkdir(self.directory) # Create directory if not found
|
|
|
|
else:
|
|
|
|
raise Exception("Directory not exists: %s" % self.directory)
|
|
|
|
self.content = None # Load content.json
|
|
|
|
self.peers = {} # Key: ip:port, Value: Peer.Peer
|
|
|
|
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
|
|
|
|
self.last_announce = 0 # Last announce time to tracker
|
|
|
|
self.worker_manager = WorkerManager(self) # Handle site download from other peers
|
2015-01-18 22:52:19 +01:00
|
|
|
self.bad_files = {} # SHA512 check failed files, need to redownload
|
2015-01-12 02:03:45 +01:00
|
|
|
self.content_updated = None # Content.js update time
|
|
|
|
self.last_downloads = [] # Files downloaded in run of self.download()
|
|
|
|
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
|
|
|
|
self.page_requested = False # Page viewed in browser
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.content_manager = ContentManager(self) # Load contents
|
2015-01-12 02:03:45 +01:00
|
|
|
self.loadSettings() # Load settings from sites.json
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
|
2015-01-18 22:52:19 +01:00
|
|
|
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
|
2015-01-12 02:03:45 +01:00
|
|
|
self.log.debug("New auth key: %s" % self.settings["auth_key"])
|
|
|
|
self.saveSettings()
|
2015-01-18 22:52:19 +01:00
|
|
|
|
|
|
|
if not self.settings.get("wrapper_key"): # To auth websocket permissions
|
|
|
|
self.settings["wrapper_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
|
|
|
|
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
|
|
|
|
self.saveSettings()
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
self.websockets = [] # Active site websocket connections
|
|
|
|
|
|
|
|
# Add event listeners
|
|
|
|
self.addEventListeners()
|
|
|
|
|
|
|
|
|
|
|
|
# Load site settings from data/sites.json
|
|
|
|
def loadSettings(self):
|
|
|
|
sites_settings = json.load(open("data/sites.json"))
|
|
|
|
if self.address in sites_settings:
|
|
|
|
self.settings = sites_settings[self.address]
|
|
|
|
else:
|
|
|
|
if self.address == config.homepage: # Add admin permissions to homepage
|
|
|
|
permissions = ["ADMIN"]
|
|
|
|
else:
|
|
|
|
permissions = []
|
|
|
|
self.settings = { "own": False, "serving": True, "permissions": permissions } # Default
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
# Save site settings to data/sites.json
|
|
|
|
def saveSettings(self):
|
|
|
|
sites_settings = json.load(open("data/sites.json"))
|
|
|
|
sites_settings[self.address] = self.settings
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
|
2015-01-12 02:03:45 +01:00
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
# Sercurity check and return path of site's file
|
|
|
|
def getPath(self, inner_path):
|
|
|
|
inner_path = inner_path.replace("\\", "/") # Windows separator fix
|
|
|
|
inner_path = re.sub("^%s/" % re.escape(self.directory), "", inner_path) # Remove site directory if begins with it
|
|
|
|
file_path = self.directory+"/"+inner_path
|
|
|
|
allowed_dir = os.path.abspath(self.directory) # Only files within this directory allowed
|
|
|
|
if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir):
|
|
|
|
raise Exception("File not allowed: %s" % file_path)
|
|
|
|
return file_path
|
|
|
|
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
# Download all file from content.json
|
|
|
|
@util.Noparallel(blocking=True)
|
|
|
|
def downloadContent(self, inner_path, download_files=True, peer=None):
|
|
|
|
s = time.time()
|
|
|
|
self.log.debug("Downloading %s..." % inner_path)
|
|
|
|
self.last_downloads.append(inner_path)
|
|
|
|
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
|
|
|
|
content_inner_dir = self.content_manager.toDir(inner_path)
|
|
|
|
if not found: return False # Could not download content.json
|
|
|
|
|
|
|
|
self.log.debug("Got %s" % inner_path)
|
|
|
|
changed = self.content_manager.loadContent(inner_path, load_includes=False)
|
|
|
|
|
|
|
|
# Start download files
|
|
|
|
evts = []
|
|
|
|
if download_files:
|
|
|
|
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
|
|
|
|
file_inner_path = content_inner_dir+file_relative_path
|
|
|
|
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) # No waiting for finish, return the event
|
|
|
|
if res != True: # Need downloading
|
|
|
|
self.last_downloads.append(file_inner_path)
|
|
|
|
evts.append(res) # Append evt
|
|
|
|
|
|
|
|
# Wait for includes download
|
|
|
|
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
|
|
|
|
file_inner_path = content_inner_dir+file_relative_path
|
|
|
|
self.downloadContent(file_inner_path, download_files=download_files, peer=peer)
|
|
|
|
|
|
|
|
self.log.debug("%s: Includes downloaded" % inner_path)
|
|
|
|
self.log.debug("%s: Downloading %s files..." % (inner_path, len(evts)))
|
|
|
|
gevent.joinall(evts)
|
|
|
|
self.log.debug("%s: All file downloaded in %.2fs" % (inner_path, time.time()-s))
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
# Download all files of the site
|
2015-01-12 02:03:45 +01:00
|
|
|
@util.Noparallel(blocking=False)
|
|
|
|
def download(self):
|
2015-01-17 18:50:56 +01:00
|
|
|
self.log.debug("Start downloading...%s" % self.bad_files)
|
2015-01-12 02:03:45 +01:00
|
|
|
self.announce()
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.last_downloads = []
|
|
|
|
found = self.downloadContent("content.json")
|
|
|
|
|
|
|
|
return found
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Update content.json from peers and download changed files
|
|
|
|
@util.Noparallel()
|
|
|
|
def update(self):
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.content_manager.loadContent("content.json") # Reload content.json
|
2015-01-12 02:03:45 +01:00
|
|
|
self.content_updated = None
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
# Download all content.json again
|
|
|
|
for inner_path in self.content_manager.contents.keys():
|
|
|
|
self.needFile(inner_path, update=True)
|
|
|
|
changed = self.content_manager.loadContent("content.json")
|
|
|
|
if changed:
|
|
|
|
for changed_file in changed:
|
2015-01-12 02:03:45 +01:00
|
|
|
self.bad_files[changed_file] = True
|
2015-01-24 19:14:29 +01:00
|
|
|
if not self.settings["own"]: self.checkFiles(quick_check=True) # Quick check files based on file size
|
2015-01-12 02:03:45 +01:00
|
|
|
if self.bad_files:
|
|
|
|
self.download()
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
return changed
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
2015-02-10 00:08:25 +01:00
|
|
|
def publisher(self, inner_path, peers, published, limit):
|
|
|
|
timeout = 5+int(os.path.getsize(self.getPath(inner_path))/1024) # Timeout: 5sec + size in kb
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
while 1:
|
|
|
|
if not peers or len(published) >= limit: break # All peers done, or published engouht
|
|
|
|
peer = peers.pop(0)
|
2015-01-12 02:03:45 +01:00
|
|
|
result = {"exception": "Timeout"}
|
|
|
|
try:
|
2015-02-10 00:08:25 +01:00
|
|
|
with gevent.Timeout(timeout, False):
|
2015-01-12 02:03:45 +01:00
|
|
|
result = peer.sendCmd("update", {
|
|
|
|
"site": self.address,
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
"inner_path": inner_path,
|
|
|
|
"body": open(self.getPath(inner_path), "rb").read(),
|
2015-01-12 02:03:45 +01:00
|
|
|
"peer": (config.ip_external, config.fileserver_port)
|
|
|
|
})
|
|
|
|
except Exception, err:
|
2015-01-17 18:50:56 +01:00
|
|
|
result = {"exception": Debug.formatException(err)}
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
if result and "ok" in result:
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
published.append(peer)
|
|
|
|
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
|
2015-01-12 02:03:45 +01:00
|
|
|
else:
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.log.info("[ERROR] %s: %s" % (peer.key, result))
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Update content.json on peers
|
|
|
|
def publish(self, limit=3, inner_path="content.json"):
|
|
|
|
self.log.info( "Publishing to %s/%s peers..." % (limit, len(self.peers)) )
|
|
|
|
published = [] # Successfuly published (Peer)
|
|
|
|
publishers = [] # Publisher threads
|
|
|
|
peers = self.peers.values()
|
|
|
|
for i in range(limit):
|
|
|
|
publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit)
|
|
|
|
publishers.append(publisher)
|
|
|
|
|
|
|
|
gevent.joinall(publishers) # Wait for all publishers
|
|
|
|
|
|
|
|
self.log.info("Successfuly published to %s peers" % len(published))
|
|
|
|
return len(published)
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Check and download if file not exits
|
2015-01-14 02:41:13 +01:00
|
|
|
def needFile(self, inner_path, update=False, blocking=True, peer=None, priority=0):
|
2015-01-12 02:03:45 +01:00
|
|
|
if os.path.isfile(self.getPath(inner_path)) and not update: # File exits, no need to do anything
|
|
|
|
return True
|
|
|
|
elif self.settings["serving"] == False: # Site not serving
|
|
|
|
return False
|
|
|
|
else: # Wait until file downloaded
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not self.content_manager.contents.get("content.json"): # No content.json, download it first!
|
2015-01-12 02:03:45 +01:00
|
|
|
self.log.debug("Need content.json first")
|
|
|
|
self.announce()
|
|
|
|
if inner_path != "content.json": # Prevent double download
|
2015-01-14 22:57:43 +01:00
|
|
|
task = self.worker_manager.addTask("content.json", peer)
|
2015-01-12 02:03:45 +01:00
|
|
|
task.get()
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.content_manager.loadContent()
|
|
|
|
if not self.content_manager.contents.get("content.json"): return False # Content.json download failed
|
|
|
|
|
|
|
|
if not inner_path.endswith("content.json") and not self.content_manager.getFileInfo(inner_path): # No info for file, download all content.json first
|
|
|
|
self.log.debug("No info for %s, waiting for all content.json" % inner_path)
|
|
|
|
success = self.downloadContent("content.json", download_files=False)
|
|
|
|
if not success: return False
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-01-14 02:41:13 +01:00
|
|
|
task = self.worker_manager.addTask(inner_path, peer, priority=priority)
|
2015-01-12 02:03:45 +01:00
|
|
|
if blocking:
|
|
|
|
return task.get()
|
|
|
|
else:
|
|
|
|
return task
|
|
|
|
|
|
|
|
|
|
|
|
# Add or update a peer to site
|
|
|
|
def addPeer(self, ip, port, return_peer = False):
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not ip: return False
|
2015-01-12 02:03:45 +01:00
|
|
|
key = "%s:%s" % (ip, port)
|
|
|
|
if key in self.peers: # Already has this ip
|
|
|
|
self.peers[key].found()
|
|
|
|
if return_peer: # Always return peer
|
|
|
|
return self.peers[key]
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
else: # New peer
|
2015-01-13 21:19:40 +01:00
|
|
|
peer = Peer(ip, port, self)
|
2015-01-12 02:03:45 +01:00
|
|
|
self.peers[key] = peer
|
|
|
|
return peer
|
|
|
|
|
|
|
|
|
|
|
|
# Add myself and get other peers from tracker
|
|
|
|
def announce(self, force=False):
|
|
|
|
if time.time() < self.last_announce+15 and not force: return # No reannouncing within 15 secs
|
|
|
|
self.last_announce = time.time()
|
2015-02-10 00:08:25 +01:00
|
|
|
errors = []
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
for protocol, ip, port in SiteManager.TRACKERS:
|
|
|
|
if protocol == "udp":
|
2015-02-10 00:08:25 +01:00
|
|
|
# self.log.debug("Announcing to %s://%s:%s..." % (protocol, ip, port))
|
2015-01-12 02:03:45 +01:00
|
|
|
tracker = UdpTrackerClient(ip, port)
|
|
|
|
tracker.peer_port = config.fileserver_port
|
|
|
|
try:
|
|
|
|
tracker.connect()
|
|
|
|
tracker.poll_once()
|
2015-01-15 23:24:51 +01:00
|
|
|
tracker.announce(info_hash=hashlib.sha1(self.address).hexdigest(), num_want=50)
|
2015-01-12 02:03:45 +01:00
|
|
|
back = tracker.poll_once()
|
|
|
|
peers = back["response"]["peers"]
|
2015-01-17 18:50:56 +01:00
|
|
|
except Exception, err:
|
2015-02-10 00:08:25 +01:00
|
|
|
errors.append("%s://%s:%s" % (protocol, ip, port))
|
2015-01-17 18:50:56 +01:00
|
|
|
continue
|
|
|
|
|
|
|
|
added = 0
|
|
|
|
for peer in peers:
|
|
|
|
if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself)
|
|
|
|
continue
|
|
|
|
if self.addPeer(peer["addr"], peer["port"]): added += 1
|
|
|
|
if added:
|
|
|
|
self.worker_manager.onPeers()
|
|
|
|
self.updateWebsocket(peers_added=added)
|
2015-01-30 18:46:48 +01:00
|
|
|
self.log.debug("Found %s peers, new: %s" % (len(peers), added))
|
2015-01-12 02:03:45 +01:00
|
|
|
else:
|
|
|
|
pass # TODO: http tracker support
|
2015-01-30 18:46:48 +01:00
|
|
|
|
2015-02-10 00:08:25 +01:00
|
|
|
if len(errors) < len(SiteManager.TRACKERS): # Less errors than total tracker nums
|
|
|
|
self.log.debug("Announced to %s trackers, errors: %s" % (len(SiteManager.TRACKERS), errors))
|
2015-01-30 18:46:48 +01:00
|
|
|
else:
|
|
|
|
self.log.error("Announced to %s trackers, failed" % len(SiteManager.TRACKERS))
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Check and try to fix site files integrity
|
|
|
|
def checkFiles(self, quick_check=True):
|
|
|
|
self.log.debug("Checking files... Quick:%s" % quick_check)
|
|
|
|
bad_files = self.verifyFiles(quick_check)
|
|
|
|
if bad_files:
|
|
|
|
for bad_file in bad_files:
|
|
|
|
self.bad_files[bad_file] = True
|
|
|
|
|
|
|
|
|
2015-01-21 12:58:26 +01:00
|
|
|
def deleteFiles(self):
|
|
|
|
self.log.debug("Deleting files from content.json...")
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
files = [] # Get filenames
|
|
|
|
for content_inner_path, content in self.content_manager.contents.items():
|
|
|
|
files.append(content_inner_path)
|
|
|
|
for file_relative_path in content["files"].keys():
|
|
|
|
file_inner_path = self.content_manager.toDir(content_inner_path)+file_relative_path # Relative to content.json
|
|
|
|
files.append(file_inner_path)
|
|
|
|
|
2015-01-21 12:58:26 +01:00
|
|
|
for inner_path in files:
|
|
|
|
path = self.getPath(inner_path)
|
|
|
|
if os.path.isfile(path): os.unlink(path)
|
|
|
|
|
|
|
|
self.log.debug("Deleting empty dirs...")
|
|
|
|
for root, dirs, files in os.walk(self.directory, topdown=False):
|
|
|
|
for dir in dirs:
|
|
|
|
path = os.path.join(root,dir)
|
|
|
|
if os.path.isdir(path) and os.listdir(path) == []:
|
|
|
|
os.removedirs(path)
|
|
|
|
self.log.debug("Removing %s" % path)
|
|
|
|
if os.path.isdir(self.directory) and os.listdir(self.directory) == []: os.removedirs(self.directory) # Remove sites directory if empty
|
|
|
|
|
|
|
|
if os.path.isdir(self.directory):
|
|
|
|
self.log.debug("Some unknown file remained in site data dir: %s..." % self.directory)
|
|
|
|
return False # Some files not deleted
|
|
|
|
else:
|
|
|
|
self.log.debug("Site data directory deleted: %s..." % self.directory)
|
|
|
|
return True # All clean
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# - Events -
|
|
|
|
|
|
|
|
# Add event listeners
|
|
|
|
def addEventListeners(self):
|
|
|
|
self.onFileStart = util.Event() # If WorkerManager added new task
|
|
|
|
self.onFileDone = util.Event() # If WorkerManager successfuly downloaded a file
|
|
|
|
self.onFileFail = util.Event() # If WorkerManager failed to download a file
|
|
|
|
self.onComplete = util.Event() # All file finished
|
|
|
|
|
|
|
|
self.onFileStart.append(lambda inner_path: self.fileStarted()) # No parameters to make Noparallel batching working
|
|
|
|
self.onFileDone.append(lambda inner_path: self.fileDone(inner_path))
|
|
|
|
self.onFileFail.append(lambda inner_path: self.fileFailed(inner_path))
|
|
|
|
|
|
|
|
|
|
|
|
# Send site status update to websocket clients
|
|
|
|
def updateWebsocket(self, **kwargs):
|
|
|
|
if kwargs:
|
|
|
|
param = {"event": kwargs.items()[0]}
|
|
|
|
else:
|
|
|
|
param = None
|
|
|
|
for ws in self.websockets:
|
|
|
|
ws.event("siteChanged", self, param)
|
|
|
|
|
|
|
|
|
|
|
|
# File download started
|
|
|
|
@util.Noparallel(blocking=False)
|
|
|
|
def fileStarted(self):
|
|
|
|
time.sleep(0.001) # Wait for other files adds
|
|
|
|
self.updateWebsocket(file_started=True)
|
|
|
|
|
|
|
|
|
|
|
|
# File downloaded successful
|
|
|
|
def fileDone(self, inner_path):
|
|
|
|
# File downloaded, remove it from bad files
|
|
|
|
if inner_path in self.bad_files:
|
2015-01-12 19:11:35 +01:00
|
|
|
self.log.info("Bad file solved: %s" % inner_path)
|
2015-01-12 02:03:45 +01:00
|
|
|
del(self.bad_files[inner_path])
|
|
|
|
|
|
|
|
# Update content.json last downlad time
|
|
|
|
if inner_path == "content.json":
|
|
|
|
self.content_updated = time.time()
|
|
|
|
|
|
|
|
self.updateWebsocket(file_done=inner_path)
|
|
|
|
|
|
|
|
|
|
|
|
# File download failed
|
|
|
|
def fileFailed(self, inner_path):
|
|
|
|
if inner_path == "content.json":
|
|
|
|
self.content_updated = False
|
|
|
|
self.log.error("Can't update content.json")
|
|
|
|
|
|
|
|
self.updateWebsocket(file_failed=inner_path)
|
|
|
|
|
|
|
|
|
2015-01-18 22:52:19 +01:00
|
|
|
# Verify all files sha512sum using content.json
|
2015-01-12 02:03:45 +01:00
|
|
|
def verifyFiles(self, quick_check=False): # Fast = using file size
|
|
|
|
bad_files = []
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not self.content_manager.contents.get("content.json"): # No content.json, download it first
|
2015-01-12 02:03:45 +01:00
|
|
|
self.needFile("content.json", update=True) # Force update to fix corrupt file
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.content_manager.loadContent() # Reload content.json
|
|
|
|
for content_inner_path, content in self.content_manager.contents.items():
|
|
|
|
for file_relative_path in content["files"].keys():
|
|
|
|
file_inner_path = self.content_manager.toDir(content_inner_path)+file_relative_path # Relative to content.json
|
|
|
|
file_inner_path = file_inner_path.strip("/") # Strip leading /
|
|
|
|
file_path = self.getPath(file_inner_path)
|
|
|
|
if not os.path.isfile(file_path):
|
|
|
|
self.log.error("[MISSING] %s" % file_inner_path)
|
|
|
|
bad_files.append(file_inner_path)
|
|
|
|
continue
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if quick_check:
|
|
|
|
ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"]
|
|
|
|
else:
|
|
|
|
ok = self.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not ok:
|
|
|
|
self.log.error("[ERROR] %s" % file_inner_path)
|
|
|
|
bad_files.append(file_inner_path)
|
|
|
|
self.log.debug("%s verified: %s files, quick_check: %s, bad files: %s" % (content_inner_path, len(content["files"]), quick_check, bad_files))
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
return bad_files
|