2015-01-12 02:03:45 +01:00
|
|
|
import os, json, logging, hashlib, re, time, string, random
|
|
|
|
from lib.subtl.subtl import UdpTrackerClient
|
|
|
|
import gevent
|
|
|
|
import util
|
|
|
|
from Config import config
|
|
|
|
from Peer import Peer
|
|
|
|
from Worker import WorkerManager
|
|
|
|
from Crypt import CryptHash
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
from Content import ContentManager
|
2015-03-19 21:19:14 +01:00
|
|
|
from SiteStorage import SiteStorage
|
2015-01-12 02:03:45 +01:00
|
|
|
import SiteManager
|
|
|
|
|
|
|
|
class Site:
|
|
|
|
def __init__(self, address, allow_create=True):
|
|
|
|
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
|
|
|
|
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
|
|
|
|
self.log = logging.getLogger("Site:%s" % self.address_short)
|
|
|
|
|
|
|
|
self.content = None # Load content.json
|
|
|
|
self.peers = {} # Key: ip:port, Value: Peer.Peer
|
|
|
|
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
|
|
|
|
self.last_announce = 0 # Last announce time to tracker
|
|
|
|
self.worker_manager = WorkerManager(self) # Handle site download from other peers
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.bad_files = {} # SHA512 check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
|
2015-01-12 02:03:45 +01:00
|
|
|
self.content_updated = None # Content.js update time
|
|
|
|
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
|
|
|
|
self.page_requested = False # Page viewed in browser
|
|
|
|
|
2015-03-19 21:19:14 +01:00
|
|
|
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
|
2015-01-12 02:03:45 +01:00
|
|
|
self.loadSettings() # Load settings from sites.json
|
2015-02-14 14:05:00 +01:00
|
|
|
self.content_manager = ContentManager(self) # Load contents
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
|
2015-01-18 22:52:19 +01:00
|
|
|
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
|
2015-01-12 02:03:45 +01:00
|
|
|
self.log.debug("New auth key: %s" % self.settings["auth_key"])
|
|
|
|
self.saveSettings()
|
2015-01-18 22:52:19 +01:00
|
|
|
|
|
|
|
if not self.settings.get("wrapper_key"): # To auth websocket permissions
|
|
|
|
self.settings["wrapper_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
|
|
|
|
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
|
|
|
|
self.saveSettings()
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
self.websockets = [] # Active site websocket connections
|
|
|
|
|
|
|
|
# Add event listeners
|
|
|
|
self.addEventListeners()
|
|
|
|
|
|
|
|
|
2015-03-11 01:12:53 +01:00
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "Site %s" % self.address_short
|
|
|
|
|
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return "<%s>" % self.__str__()
|
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# Load site settings from data/sites.json
|
|
|
|
def loadSettings(self):
|
|
|
|
sites_settings = json.load(open("data/sites.json"))
|
|
|
|
if self.address in sites_settings:
|
|
|
|
self.settings = sites_settings[self.address]
|
|
|
|
else:
|
|
|
|
if self.address == config.homepage: # Add admin permissions to homepage
|
|
|
|
permissions = ["ADMIN"]
|
|
|
|
else:
|
|
|
|
permissions = []
|
|
|
|
self.settings = { "own": False, "serving": True, "permissions": permissions } # Default
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
# Save site settings to data/sites.json
|
|
|
|
def saveSettings(self):
|
|
|
|
sites_settings = json.load(open("data/sites.json"))
|
|
|
|
sites_settings[self.address] = self.settings
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
|
2015-01-12 02:03:45 +01:00
|
|
|
return
|
|
|
|
|
|
|
|
|
2015-02-14 14:05:00 +01:00
|
|
|
# Max site size in MB
|
|
|
|
def getSizeLimit(self):
|
|
|
|
return self.settings.get("size_limit", config.size_limit)
|
|
|
|
|
|
|
|
|
|
|
|
# Next size limit based on current size
|
|
|
|
def getNextSizeLimit(self):
|
|
|
|
size_limits = [10,20,50,100,200,500,1000,2000,5000,10000,20000,50000,100000]
|
|
|
|
size = self.settings.get("size", 0)
|
|
|
|
for size_limit in size_limits:
|
|
|
|
if size*1.2 < size_limit*1024*1024:
|
|
|
|
return size_limit
|
|
|
|
return 999999
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
# Download all file from content.json
|
|
|
|
@util.Noparallel(blocking=True)
|
|
|
|
def downloadContent(self, inner_path, download_files=True, peer=None):
|
|
|
|
s = time.time()
|
|
|
|
self.log.debug("Downloading %s..." % inner_path)
|
|
|
|
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
|
|
|
|
content_inner_dir = self.content_manager.toDir(inner_path)
|
|
|
|
if not found: return False # Could not download content.json
|
|
|
|
|
|
|
|
self.log.debug("Got %s" % inner_path)
|
|
|
|
changed = self.content_manager.loadContent(inner_path, load_includes=False)
|
|
|
|
|
|
|
|
# Start download files
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
file_threads = []
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if download_files:
|
|
|
|
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
|
|
|
|
file_inner_path = content_inner_dir+file_relative_path
|
|
|
|
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) # No waiting for finish, return the event
|
|
|
|
if res != True: # Need downloading
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
file_threads.append(res) # Append evt
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
|
|
|
# Wait for includes download
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
include_threads = []
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
|
|
|
|
file_inner_path = content_inner_dir+file_relative_path
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
|
|
|
|
include_threads.append(include_thread)
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
|
|
|
|
gevent.joinall(include_threads)
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.log.debug("%s: Includes downloaded" % inner_path)
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
|
2015-02-26 01:32:27 +01:00
|
|
|
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
gevent.joinall(file_threads)
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.log.debug("%s: All file downloaded in %.2fs" % (inner_path, time.time()-s))
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
# Return bad files with less than 3 retry
|
|
|
|
def getReachableBadFiles(self):
|
|
|
|
if not self.bad_files: return False
|
|
|
|
return [bad_file for bad_file, retry in self.bad_files.iteritems() if retry < 3]
|
|
|
|
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
# Download all files of the site
|
2015-01-12 02:03:45 +01:00
|
|
|
@util.Noparallel(blocking=False)
|
2015-02-14 14:05:00 +01:00
|
|
|
def download(self, check_size=False):
|
2015-01-17 18:50:56 +01:00
|
|
|
self.log.debug("Start downloading...%s" % self.bad_files)
|
2015-01-12 02:03:45 +01:00
|
|
|
self.announce()
|
2015-02-14 14:05:00 +01:00
|
|
|
if check_size: # Check the size first
|
|
|
|
valid = downloadContent(download_files=False)
|
|
|
|
if not valid: return False # Cant download content.jsons or size is not fits
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
found = self.downloadContent("content.json")
|
|
|
|
|
|
|
|
return found
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Update content.json from peers and download changed files
|
|
|
|
@util.Noparallel()
|
|
|
|
def update(self):
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.content_manager.loadContent("content.json") # Reload content.json
|
2015-01-12 02:03:45 +01:00
|
|
|
self.content_updated = None
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
# Download all content.json again
|
2015-02-26 01:32:27 +01:00
|
|
|
content_threads = []
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
for inner_path in self.content_manager.contents.keys():
|
2015-02-26 01:32:27 +01:00
|
|
|
content_threads.append(self.needFile(inner_path, update=True, blocking=False))
|
|
|
|
|
|
|
|
self.log.debug("Waiting %s content.json to finish..." % len(content_threads))
|
|
|
|
gevent.joinall(content_threads)
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
changed = self.content_manager.loadContent("content.json")
|
|
|
|
if changed:
|
|
|
|
for changed_file in changed:
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.bad_files[changed_file] = self.bad_files.get(changed_file, 0)+1
|
2015-03-19 21:19:14 +01:00
|
|
|
if not self.settings["own"]: self.storage.checkFiles(quick_check=True) # Quick check files based on file size
|
2015-01-12 02:03:45 +01:00
|
|
|
if self.bad_files:
|
|
|
|
self.download()
|
2015-02-14 14:05:00 +01:00
|
|
|
|
|
|
|
self.settings["size"] = self.content_manager.getTotalSize() # Update site size
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
return changed
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
2015-03-19 21:19:14 +01:00
|
|
|
# Publish worker
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
def publisher(self, inner_path, peers, published, limit, event_done):
|
2015-03-19 21:19:14 +01:00
|
|
|
timeout = 5+int(self.storage.getSize(inner_path)/1024) # Timeout: 5sec + size in kb
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
while 1:
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
if not peers or len(published) >= limit:
|
|
|
|
event_done.set(True)
|
|
|
|
break # All peers done, or published engouht
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
peer = peers.pop(0)
|
2015-01-12 02:03:45 +01:00
|
|
|
result = {"exception": "Timeout"}
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
|
|
|
|
for retry in range(2):
|
|
|
|
try:
|
|
|
|
with gevent.Timeout(timeout, False):
|
|
|
|
result = peer.request("update", {
|
|
|
|
"site": self.address,
|
|
|
|
"inner_path": inner_path,
|
2015-03-19 21:19:14 +01:00
|
|
|
"body": self.storage.open(inner_path).read(),
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
"peer": (config.ip_external, config.fileserver_port)
|
|
|
|
})
|
|
|
|
if result: break
|
|
|
|
except Exception, err:
|
|
|
|
result = {"exception": Debug.formatException(err)}
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
if result and "ok" in result:
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
published.append(peer)
|
|
|
|
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
|
2015-01-12 02:03:45 +01:00
|
|
|
else:
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.log.info("[ERROR] %s: %s" % (peer.key, result))
|
|
|
|
|
|
|
|
|
|
|
|
# Update content.json on peers
|
2015-04-03 02:47:51 +02:00
|
|
|
def publish(self, limit=5, inner_path="content.json"):
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.log.info( "Publishing to %s/%s peers..." % (limit, len(self.peers)) )
|
|
|
|
published = [] # Successfuly published (Peer)
|
|
|
|
publishers = [] # Publisher threads
|
|
|
|
peers = self.peers.values()
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
|
|
|
|
random.shuffle(peers)
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
event_done = gevent.event.AsyncResult()
|
2015-03-31 01:01:39 +02:00
|
|
|
for i in range(min(len(self.peers), limit, 5)): # Max 5 thread
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit, event_done)
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
publishers.append(publisher)
|
|
|
|
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
event_done.get() # Wait for done
|
|
|
|
if len(published) < min(len(self.peers), limit): time.sleep(0.2) # If less than we need sleep a bit
|
2015-04-03 14:31:15 +02:00
|
|
|
if len(published) == 0: gevent.joinall(publishers) # No successful publish, wait for all publisher
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
|
|
|
|
self.log.info("Successfuly published to %s peers" % len(published))
|
|
|
|
return len(published)
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Check and download if file not exits
|
2015-01-14 02:41:13 +01:00
|
|
|
def needFile(self, inner_path, update=False, blocking=True, peer=None, priority=0):
|
2015-03-19 21:19:14 +01:00
|
|
|
if self.storage.isFile(inner_path) and not update: # File exits, no need to do anything
|
2015-01-12 02:03:45 +01:00
|
|
|
return True
|
|
|
|
elif self.settings["serving"] == False: # Site not serving
|
|
|
|
return False
|
|
|
|
else: # Wait until file downloaded
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
self.bad_files[inner_path] = True # Mark as bad file
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not self.content_manager.contents.get("content.json"): # No content.json, download it first!
|
2015-01-12 02:03:45 +01:00
|
|
|
self.log.debug("Need content.json first")
|
|
|
|
self.announce()
|
|
|
|
if inner_path != "content.json": # Prevent double download
|
2015-01-14 22:57:43 +01:00
|
|
|
task = self.worker_manager.addTask("content.json", peer)
|
2015-01-12 02:03:45 +01:00
|
|
|
task.get()
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
self.content_manager.loadContent()
|
|
|
|
if not self.content_manager.contents.get("content.json"): return False # Content.json download failed
|
|
|
|
|
|
|
|
if not inner_path.endswith("content.json") and not self.content_manager.getFileInfo(inner_path): # No info for file, download all content.json first
|
|
|
|
self.log.debug("No info for %s, waiting for all content.json" % inner_path)
|
|
|
|
success = self.downloadContent("content.json", download_files=False)
|
|
|
|
if not success: return False
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-01-14 02:41:13 +01:00
|
|
|
task = self.worker_manager.addTask(inner_path, peer, priority=priority)
|
2015-01-12 02:03:45 +01:00
|
|
|
if blocking:
|
|
|
|
return task.get()
|
|
|
|
else:
|
|
|
|
return task
|
|
|
|
|
|
|
|
|
|
|
|
# Add or update a peer to site
|
|
|
|
def addPeer(self, ip, port, return_peer = False):
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if not ip: return False
|
2015-01-12 02:03:45 +01:00
|
|
|
key = "%s:%s" % (ip, port)
|
|
|
|
if key in self.peers: # Already has this ip
|
|
|
|
self.peers[key].found()
|
|
|
|
if return_peer: # Always return peer
|
|
|
|
return self.peers[key]
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
else: # New peer
|
2015-01-13 21:19:40 +01:00
|
|
|
peer = Peer(ip, port, self)
|
2015-01-12 02:03:45 +01:00
|
|
|
self.peers[key] = peer
|
|
|
|
return peer
|
|
|
|
|
|
|
|
|
|
|
|
# Add myself and get other peers from tracker
|
|
|
|
def announce(self, force=False):
|
|
|
|
if time.time() < self.last_announce+15 and not force: return # No reannouncing within 15 secs
|
|
|
|
self.last_announce = time.time()
|
2015-02-10 00:08:25 +01:00
|
|
|
errors = []
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
for protocol, ip, port in SiteManager.TRACKERS:
|
|
|
|
if protocol == "udp":
|
2015-02-10 00:08:25 +01:00
|
|
|
# self.log.debug("Announcing to %s://%s:%s..." % (protocol, ip, port))
|
2015-01-12 02:03:45 +01:00
|
|
|
tracker = UdpTrackerClient(ip, port)
|
|
|
|
tracker.peer_port = config.fileserver_port
|
|
|
|
try:
|
|
|
|
tracker.connect()
|
|
|
|
tracker.poll_once()
|
2015-01-15 23:24:51 +01:00
|
|
|
tracker.announce(info_hash=hashlib.sha1(self.address).hexdigest(), num_want=50)
|
2015-01-12 02:03:45 +01:00
|
|
|
back = tracker.poll_once()
|
|
|
|
peers = back["response"]["peers"]
|
2015-01-17 18:50:56 +01:00
|
|
|
except Exception, err:
|
2015-02-10 00:08:25 +01:00
|
|
|
errors.append("%s://%s:%s" % (protocol, ip, port))
|
2015-01-17 18:50:56 +01:00
|
|
|
continue
|
|
|
|
|
|
|
|
added = 0
|
|
|
|
for peer in peers:
|
|
|
|
if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself)
|
|
|
|
continue
|
|
|
|
if self.addPeer(peer["addr"], peer["port"]): added += 1
|
|
|
|
if added:
|
|
|
|
self.worker_manager.onPeers()
|
|
|
|
self.updateWebsocket(peers_added=added)
|
2015-02-14 14:05:00 +01:00
|
|
|
self.settings["peers"] = len(peers)
|
|
|
|
self.saveSettings()
|
2015-01-30 18:46:48 +01:00
|
|
|
self.log.debug("Found %s peers, new: %s" % (len(peers), added))
|
2015-01-12 02:03:45 +01:00
|
|
|
else:
|
|
|
|
pass # TODO: http tracker support
|
2015-01-30 18:46:48 +01:00
|
|
|
|
2015-02-10 00:08:25 +01:00
|
|
|
if len(errors) < len(SiteManager.TRACKERS): # Less errors than total tracker nums
|
|
|
|
self.log.debug("Announced to %s trackers, errors: %s" % (len(SiteManager.TRACKERS), errors))
|
2015-01-30 18:46:48 +01:00
|
|
|
else:
|
|
|
|
self.log.error("Announced to %s trackers, failed" % len(SiteManager.TRACKERS))
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
2015-04-03 02:47:51 +02:00
|
|
|
# Need open connections
|
|
|
|
def needConnections(self):
|
|
|
|
need = min(len(self.peers)/2, 10) # Connect to half of total peers, but max 10
|
|
|
|
need = max(need, 5) # But minimum 5 peers
|
|
|
|
need = min(len(self.peers), need) # Max total peers
|
|
|
|
|
|
|
|
connected = 0
|
|
|
|
for peer in self.peers.values(): # Check current connected number
|
|
|
|
if peer.connection and peer.connection.connected:
|
|
|
|
connected += 1
|
|
|
|
|
|
|
|
self.log.debug("Need connections: %s, Current: %s, Total: %s" % (need, connected, len(self.peers)))
|
|
|
|
|
|
|
|
if connected < need: # Need more than we have
|
|
|
|
for peer in self.peers.values():
|
|
|
|
if not peer.connection or not peer.connection.connected: # No peer connection or disconnected
|
|
|
|
peer.connect()
|
|
|
|
if peer.connection and peer.connection.connected: connected += 1 # Successfully connected
|
|
|
|
if connected >= need: break
|
|
|
|
return connected
|
|
|
|
|
2015-01-21 12:58:26 +01:00
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# - Events -
|
|
|
|
|
|
|
|
# Add event listeners
|
|
|
|
def addEventListeners(self):
|
|
|
|
self.onFileStart = util.Event() # If WorkerManager added new task
|
|
|
|
self.onFileDone = util.Event() # If WorkerManager successfuly downloaded a file
|
|
|
|
self.onFileFail = util.Event() # If WorkerManager failed to download a file
|
|
|
|
self.onComplete = util.Event() # All file finished
|
|
|
|
|
|
|
|
self.onFileStart.append(lambda inner_path: self.fileStarted()) # No parameters to make Noparallel batching working
|
|
|
|
self.onFileDone.append(lambda inner_path: self.fileDone(inner_path))
|
|
|
|
self.onFileFail.append(lambda inner_path: self.fileFailed(inner_path))
|
|
|
|
|
|
|
|
|
|
|
|
# Send site status update to websocket clients
|
|
|
|
def updateWebsocket(self, **kwargs):
|
|
|
|
if kwargs:
|
|
|
|
param = {"event": kwargs.items()[0]}
|
|
|
|
else:
|
|
|
|
param = None
|
|
|
|
for ws in self.websockets:
|
|
|
|
ws.event("siteChanged", self, param)
|
|
|
|
|
|
|
|
|
|
|
|
# File download started
|
|
|
|
@util.Noparallel(blocking=False)
|
|
|
|
def fileStarted(self):
|
|
|
|
time.sleep(0.001) # Wait for other files adds
|
|
|
|
self.updateWebsocket(file_started=True)
|
|
|
|
|
|
|
|
|
|
|
|
# File downloaded successful
|
|
|
|
def fileDone(self, inner_path):
|
|
|
|
# File downloaded, remove it from bad files
|
|
|
|
if inner_path in self.bad_files:
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
self.log.debug("Bad file solved: %s" % inner_path)
|
2015-01-12 02:03:45 +01:00
|
|
|
del(self.bad_files[inner_path])
|
|
|
|
|
|
|
|
# Update content.json last downlad time
|
|
|
|
if inner_path == "content.json":
|
|
|
|
self.content_updated = time.time()
|
|
|
|
|
|
|
|
self.updateWebsocket(file_done=inner_path)
|
|
|
|
|
|
|
|
|
|
|
|
# File download failed
|
|
|
|
def fileFailed(self, inner_path):
|
|
|
|
if inner_path == "content.json":
|
|
|
|
self.content_updated = False
|
|
|
|
self.log.error("Can't update content.json")
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
if inner_path in self.bad_files:
|
|
|
|
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0)+1
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
self.updateWebsocket(file_failed=inner_path)
|
|
|
|
|