Rev390, Fix sidebar error on non locatable IPs, Configurable bootstrap torrent trackers, Multi-line config file settings, Evenly distributed tracker announce to work better on passive connections, Avoid iframe sandbox escape by using nonces, Better html error messages, Display proper error on invalid startup parameters

This commit is contained in:
HelloZeroNet 2015-09-10 23:25:09 +02:00
parent eec0b22c1f
commit 0de6496f96
11 changed files with 117 additions and 58 deletions

View File

@ -376,7 +376,7 @@ class UiWebsocketPlugin(object):
else:
loc = geodb.get(peer.ip)
loc_cache[peer.ip] = loc
if not loc:
if not loc or "location" not in loc:
continue
# Create position array

View File

@ -8,7 +8,7 @@ class Config(object):
def __init__(self, argv):
self.version = "0.3.2"
self.rev = 378
self.rev = 390
self.argv = argv
self.action = None
self.createParser()
@ -29,6 +29,14 @@ class Config(object):
# Create command line arguments
def createArguments(self):
trackers = [
"udp://open.demonii.com:1337",
"udp://tracker.leechers-paradise.org:6969",
"udp://9.rarbg.com:2710",
"http://tracker.aletorrenty.pl:2710/announce",
"http://retracker.telecom.kz/announce",
"http://torrent.gresille.org/announce"
]
# Platform specific
if sys.platform.startswith("win"):
coffeescript = "type %s | tools\\coffee\\coffee.cmd"
@ -122,7 +130,8 @@ class Config(object):
self.parser.add_argument('--fileserver_port', help='FileServer bind port', default=15441, type=int, metavar='port')
self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
self.parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip')
self.parser.add_argument('--ip_external', help='Set reported external ip (tested on start if None)', metavar='ip')
self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*')
self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup',
type='bool', choices=[True, False], default=use_openssl)
self.parser.add_argument('--disable_encryption', help='Disable connection encryption', action='store_true')
@ -238,7 +247,8 @@ class Config(object):
if section != "global": # If not global prefix key with section
key = section + "_" + key
if val:
argv.insert(1, val)
for line in val.strip().split("\n"): # Allow multi-line values
argv.insert(1, line)
argv.insert(1, "--%s" % key)
return argv

View File

@ -170,11 +170,15 @@ class FileServer(ConnectionServer):
# Announce sites every 20 min
def announceSites(self):
import gc
first_announce = True # First start
while 1:
time.sleep(20 * 60) # Announce sites every 20 min
# Sites healthcare
for address, site in self.sites.items():
if site.settings["serving"]:
site.announce() # Announce site to tracker
if first_announce: # Announce to all trackers on startup
site.announce()
else: # If not first run only use PEX
site.announcePex()
# Reset bad file retry counter
for inner_path in site.bad_files:
@ -195,6 +199,20 @@ class FileServer(ConnectionServer):
site = None
gc.collect() # Implicit grabage collection
# Find new peers
for tracker_i in range(len(config.trackers)):
time.sleep(60 * 20 / len(config.trackers)) # Query all trackers one-by-one in 20 minutes evenly distributed
for address, site in self.sites.items():
site.announce(num = 1, pex = False)
time.sleep(2)
first_announce = False
# Detects if computer back from wakeup
def wakeupWatcher(self):
last_time = time.time()

View File

@ -38,6 +38,7 @@ class Site:
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.last_tracker_id = random.randint(0, 10) # Last announced tracker id
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
@ -510,12 +511,13 @@ class Site:
# Gather peers from tracker
# Return: Complete time or False on error
def announceTracker(self, protocol, ip, port, fileserver_port, address_hash, my_peer_id):
def announceTracker(self, protocol, address, fileserver_port, address_hash, my_peer_id):
s = time.time()
if protocol == "udp": # Udp tracker
if config.disable_udp:
return False # No udp supported
tracker = UdpTrackerClient(ip, port)
ip, port = address.split(":")
tracker = UdpTrackerClient(ip, int(port))
tracker.peer_port = fileserver_port
try:
tracker.connect()
@ -535,7 +537,7 @@ class Site:
}
req = None
try:
url = "http://" + ip + "?" + urllib.urlencode(params)
url = "http://" + address + "?" + urllib.urlencode(params)
# Load url
with gevent.Timeout(10, False): # Make sure of timeout
req = urllib2.urlopen(url, timeout=8)
@ -577,10 +579,17 @@ class Site:
return time.time() - s
# Add myself and get other peers from tracker
def announce(self, force=False):
def announce(self, force=False, num=5, pex=True):
if time.time() < self.last_announce + 30 and not force:
return # No reannouncing within 30 secs
self.last_announce = time.time()
trackers = config.trackers
if num == 1: # Only announce on one tracker, increment the queried tracker id
self.last_tracker_id += 1
self.last_tracker_id = self.last_tracker_id % len(trackers)
trackers = [trackers[self.last_tracker_id]] # We only going to use this one
errors = []
slow = []
address_hash = hashlib.sha1(self.address).hexdigest() # Site address hash
@ -595,39 +604,42 @@ class Site:
announced = 0
threads = []
for protocol, ip, port in SiteManager.TRACKERS: # Start announce threads
thread = gevent.spawn(self.announceTracker, protocol, ip, port, fileserver_port, address_hash, my_peer_id)
for tracker in trackers: # Start announce threads
protocol, address = tracker.split("://")
thread = gevent.spawn(self.announceTracker, protocol, address, fileserver_port, address_hash, my_peer_id)
threads.append(thread)
thread.ip = ip
thread.address = address
thread.protocol = protocol
if len(threads) > num: break # Announce limit
gevent.joinall(threads) # Wait for announce finish
for thread in threads:
if thread.value:
if thread.value > 1:
slow.append("%.2fs %s://%s" % (thread.value, thread.protocol, thread.ip))
slow.append("%.2fs %s://%s" % (thread.value, thread.protocol, thread.address))
announced += 1
else:
errors.append("%s://%s" % (thread.protocol, thread.ip))
errors.append("%s://%s" % (thread.protocol, thread.address))
# Save peers num
self.settings["peers"] = len(self.peers)
self.saveSettings()
if len(errors) < len(SiteManager.TRACKERS): # Less errors than total tracker nums
if len(errors) < min(num, len(trackers)): # Less errors than total tracker nums
self.log.debug(
"Announced port %s to %s trackers in %.3fs, errors: %s, slow: %s" %
(fileserver_port, announced, time.time() - s, errors, slow)
)
else:
self.log.error("Announced to %s trackers in %.3fs, failed" % (announced, time.time() - s))
self.log.error("Announce to %s trackers in %.3fs, failed" % (announced, time.time() - s))
if not [peer for peer in self.peers.values() if peer.connection and peer.connection.connected]:
# If no connected peer yet then wait for connections
gevent.spawn_later(3, self.announcePex, need_num=10) # Spawn 3 secs later
else: # Else announce immediately
self.announcePex()
if pex:
if not [peer for peer in self.peers.values() if peer.connection and peer.connection.connected]:
# If no connected peer yet then wait for connections
gevent.spawn_later(3, self.announcePex, need_num=10) # Spawn 3 secs later
else: # Else announce immediately
self.announcePex()
# Keep connections to get the updates (required for passive clients)
def needConnections(self, num=3):

View File

@ -6,26 +6,6 @@ import os
from Plugin import PluginManager
from Config import config
TRACKERS = [
("udp", "open.demonii.com", 1337),
# ("udp", "sugoi.pomf.se", 2710),
# ("udp", "tracker.coppersurfer.tk", 80),
("udp", "tracker.leechers-paradise.org", 6969),
("udp", "9.rarbg.com", 2710),
# ("udp", "www.eddie4.nl", 6969),
# ("udp", "trackr.sytes.net", 80),
# ("udp", "tracker4.piratux.com", 6969)
# ("http", "exodus.desync.com:80/announce", None), Off
("http", "tracker.aletorrenty.pl:2710/announce", None),
# ("http", "torrent.gresille.org/announce", None), # Slow
# ("http", "announce.torrentsmd.com:6969/announce", None), # Off
# ("http", "i.bandito.org/announce", None), # Off
("http", "retracker.telecom.kz/announce", None),
("http", "torrent.gresille.org/announce", None),
]
@PluginManager.acceptPlugins
class SiteManager(object):

View File

@ -4,6 +4,8 @@ import os
import mimetypes
import json
import cgi
import string
import random
from Config import config
from Site import SiteManager
@ -69,7 +71,7 @@ class UiRequest(object):
return self.actionConsole()
# Site media wrapper
else:
if self.get.get("wrapper") == "False":
if self.get.get("wrapper_nonce"):
return self.actionSiteMedia("/media" + path) # Only serve html files with frame
else:
body = self.actionWrapper(path)
@ -202,7 +204,6 @@ class UiRequest(object):
else: # Bad url
return False
def renderWrapper(self, site, path, inner_path, title, extra_headers):
file_inner_path = inner_path
if not file_inner_path:
@ -219,10 +220,12 @@ class UiRequest(object):
body_style = ""
meta_tags = ""
wrapper_nonce = self.getWrapperNonce()
if self.env.get("QUERY_STRING"):
query_string = "?" + self.env["QUERY_STRING"] + "&wrapper=False"
query_string = "?%s&wrapper_nonce=%s" % (self.env["QUERY_STRING"], wrapper_nonce)
else:
query_string = "?wrapper=False"
query_string = "?wrapper_nonce=%s" % wrapper_nonce
if self.isProxyRequest(): # Its a remote proxy request
if self.env["REMOTE_ADDR"] == "127.0.0.1": # Local client, the server address also should be 127.0.0.1
@ -260,6 +263,13 @@ class UiRequest(object):
homepage=homepage
)
# Create a new wrapper nonce that allows to get one html file without the wrapper
def getWrapperNonce(self):
wrapper_nonce = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24)
)
self.server.wrapper_nonces.append(wrapper_nonce)
return wrapper_nonce
# Returns if media request allowed from that referer
def isMediaRequestAllowed(self, site_address, referer):
@ -276,6 +286,14 @@ class UiRequest(object):
match = re.match("/media/(?P<address>[A-Za-z0-9\._-]+)/(?P<inner_path>.*)", path)
# Check wrapper nonce
content_type = self.getContentType(path)
if "htm" in content_type: # Valid nonce must present to render html files
wrapper_nonce = self.get["wrapper_nonce"]
if wrapper_nonce not in self.server.wrapper_nonces:
return self.error403("Wrapper nonce error.")
self.server.wrapper_nonces.remove(self.get["wrapper_nonce"])
referer = self.env.get("HTTP_REFERER")
if referer and match: # Only allow same site to receive media
if not self.isMediaRequestAllowed(match.group("address"), referer):
@ -421,24 +439,38 @@ class UiRequest(object):
# - Errors -
# Send bad request error
def error400(self):
def error400(self, message=""):
self.sendHeader(400)
return "Bad Request"
return self.formatError("Bad Request", message)
# You are not allowed to access this
def error403(self, message="Forbidden"):
def error403(self, message=""):
self.sendHeader(403)
return message
return self.formatError("Forbidden", message)
# Send file not found error
def error404(self, path=None):
def error404(self, path=""):
self.sendHeader(404)
return "Not Found: %s" % path.encode("utf8")
return self.formatError("Not Found", path.encode("utf8"))
# Internal server error
def error500(self, message=":("):
self.sendHeader(500)
return "<h1>Server error</h1>%s" % cgi.escape(message)
return self.formatError("Server error", cgi.escape(message))
def formatError(self, title, message):
details = {key: val for key, val in self.env.items() if hasattr(val, "endswith") and "COOKIE" not in key }
return """
<h1>%s</h1>
<h2>%s</h3>
<h3>Please <a href="https://github.com/HelloZeroNet/ZeroNet/issues">report it</a> if you think this an error.</h3>
<h4>Details:</h4>
<pre>%s</pre>
<style>
* { font-family: Consolas, Monospace; color: #333 }
pre { padding: 10px; background-color: #EEE }
</style>
""" % (title, message, json.dumps(details, indent=4))
# - Reload for eaiser developing -

View File

@ -55,6 +55,7 @@ class UiServer:
self.port = config.ui_port
if self.ip == "*":
self.ip = "" # Bind all
self.wrapper_nonces = []
self.sites = SiteManager.site_manager.list()
self.log = logging.getLogger(__name__)

View File

@ -35,7 +35,7 @@ a { color: black }
display: block; width: 80px; height: 80px; -webkit-transition: background-color 0.2s, box-shadow 0.5s; -moz-transition: background-color 0.2s, box-shadow 0.5s; -o-transition: background-color 0.2s, box-shadow 0.5s; -ms-transition: background-color 0.2s, box-shadow 0.5s; transition: background-color 0.2s, box-shadow 0.5s ; -webkit-transform: scale(0.6); -moz-transform: scale(0.6); -o-transform: scale(0.6); -ms-transform: scale(0.6); transform: scale(0.6) ; margin-left: -20px; margin-top: -20px; /* 2x size to prevent blur on anim */
/*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */
}
.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px }
.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px }
.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px }
.fixbutton-bg:hover { background-color: #AF3BFF }
.fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none }
@ -45,7 +45,7 @@ a { color: black }
.notifications { position: absolute; top: 0px; right: 80px; display: inline-block; z-index: 999; white-space: nowrap }
.notification {
position: relative; float: right; clear: both; margin: 10px; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; overflow: hidden; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; padding-bottom: 5px;
position: relative; float: right; clear: both; margin: 10px; -webkit-box-sizing: border-box; -moz-box-sizing: border-box; -o-box-sizing: border-box; -ms-box-sizing: border-box; box-sizing: border-box ; overflow: hidden; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; padding-bottom: 5px;
color: #4F4F4F; font-family: 'Lucida Grande', 'Segoe UI', Helvetica, Arial, sans-serif; font-size: 14px; line-height: 20px; /*border: 1px solid rgba(210, 206, 205, 0.2)*/
}
.notification-icon {
@ -112,7 +112,7 @@ a { color: black }
.flipper-container { width: 40px; height: 40px; position: absolute; top: 0%; left: 50%; -webkit-transform: translate3d(-50%, -50%, 0); -moz-transform: translate3d(-50%, -50%, 0); -o-transform: translate3d(-50%, -50%, 0); -ms-transform: translate3d(-50%, -50%, 0); transform: translate3d(-50%, -50%, 0) ; -webkit-perspective: 1200; -moz-perspective: 1200; -o-perspective: 1200; -ms-perspective: 1200; perspective: 1200 ; opacity: 0 }
.flipper { position: relative; display: block; height: inherit; width: inherit; -webkit-animation: flip 1.2s infinite ease-in-out; -moz-animation: flip 1.2s infinite ease-in-out; -o-animation: flip 1.2s infinite ease-in-out; -ms-animation: flip 1.2s infinite ease-in-out; animation: flip 1.2s infinite ease-in-out ; -webkit-transform-style: preserve-3d; }
.flipper .front, .flipper .back {
position: absolute; top: 0; left: 0; backface-visibility: hidden; /*transform-style: preserve-3d;*/ display: block;
position: absolute; top: 0; left: 0; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; -o-backface-visibility: hidden; -ms-backface-visibility: hidden; backface-visibility: hidden ; /*transform-style: preserve-3d;*/ display: block;
background-color: #d50000; height: 100%; width: 100%; /*outline: 1px solid transparent; /* FF AA fix */
}
.flipper .back { background-color: white; z-index: 800; -webkit-transform: rotateY(-180deg) ; -moz-transform: rotateY(-180deg) ; -o-transform: rotateY(-180deg) ; -ms-transform: rotateY(-180deg) ; transform: rotateY(-180deg) }

View File

@ -15,6 +15,7 @@
// If we are inside iframe escape from it
if (window.self !== window.top) window.open(window.location.toString(), "_top");
if (window.self !== window.top) window.stop();
if (window.self !== window.top && document.execCommand) document.execCommand("Stop", false)
// Dont allow site to load in a popup
if (window.opener) document.write("Opener not allowed")
@ -51,10 +52,13 @@ if (window.opener && window.stop) window.stop()
<!-- Site Iframe -->
<iframe src='{file_url}{query_string}' id='inner-iframe' sandbox="allow-forms allow-scripts allow-top-navigation allow-popups"></iframe>
<iframe src='about:blank' id='inner-iframe' sandbox="allow-forms allow-scripts allow-top-navigation allow-popups"></iframe>
<!-- Site info -->
<script>
console.log("Setting iframe src", document.getElementById("inner-iframe").src, "{file_url}{query_string}")
document.getElementById("inner-iframe").src = "about:blank"
document.getElementById("inner-iframe").src = "{file_url}{query_string}"
address = "{address}"
wrapper_key = "{wrapper_key}"
file_inner_path = "{file_inner_path}"

View File

@ -61,7 +61,7 @@ class WorkerManager:
elif (task["time_started"] and time.time() >= task["time_started"] + 15) or not self.workers:
# Task started more than 15 sec ago or no workers
self.log.debug("Task taking more than 15 secs, find more peers: %s" % task["inner_path"])
task["site"].announce() # Find more peers
task["site"].announce(num=1) # Find more peers
if task["peers"]: # Release the peer lock
self.log.debug("Task peer lock release: %s" % task["inner_path"])
task["peers"] = []

View File

@ -14,6 +14,8 @@ update_after_shutdown = False # If set True then update and restart zeronet aft
# Load config
from Config import config
config.parse(silent=True) # Plugins need to access the configuration
if not config.arguments: # Config parse failed, show the help screen and exit
config.parse()
# Create necessary files and dirs
if not os.path.isdir(config.log_dir):