rev338, Possible to use pure-python msgpack unpacker to save memory, Streaming file download hangup fix, Clone databases after 3 minute idle, Many site size limit related bugfixes, UiMedia served from same domain to allow ajax access, Don't allow to load resources from other domain, Site size increase ask dialog displayed again, Changed fixbutton to more Consolas-like Monaco font, Boost json files priority on download
This commit is contained in:
HelloZeroNet
parent
cd04abe521
commit
84e3f00aac
|
|
@ -8,7 +8,7 @@ class Config(object):
|
|||
|
||||
def __init__(self, argv):
|
||||
self.version = "0.3.1"
|
||||
self.rev = 330
|
||||
self.rev = 338
|
||||
self.argv = argv
|
||||
self.action = None
|
||||
self.createParser()
|
||||
|
|
@ -130,6 +130,8 @@ class Config(object):
|
|||
type='bool', choices=[True, False], default=False)
|
||||
self.parser.add_argument('--stream_downloads', help='Stream download directly to files (experimental)',
|
||||
type='bool', choices=[True, False], default=False)
|
||||
self.parser.add_argument("--msgpack_purepython", help='Use less memory, but a bit more CPU power',
|
||||
type='bool', choices=[True, False], default=False)
|
||||
|
||||
self.parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript,
|
||||
metavar='executable_path')
|
||||
|
|
|
|||
|
|
@ -237,6 +237,8 @@ class Connection(object):
|
|||
if read_bytes <= 0:
|
||||
break
|
||||
buff = self.sock.recv(16 * 1024)
|
||||
if not buff:
|
||||
break
|
||||
buff_len = len(buff)
|
||||
read_bytes -= buff_len
|
||||
file.write(buff)
|
||||
|
|
|
|||
26
src/Db/Db.py
26
src/Db/Db.py
|
|
@ -4,9 +4,23 @@ import time
|
|||
import logging
|
||||
import re
|
||||
import os
|
||||
import gevent
|
||||
|
||||
from DbCursor import DbCursor
|
||||
|
||||
opened_dbs = []
|
||||
|
||||
|
||||
# Close idle databases to save some memory
|
||||
def cleanup():
|
||||
while 1:
|
||||
time.sleep(60 * 5)
|
||||
for db in opened_dbs[:]:
|
||||
if time.time() - db.last_query_time > 60 * 3:
|
||||
db.close()
|
||||
|
||||
gevent.spawn(cleanup)
|
||||
|
||||
|
||||
class Db:
|
||||
|
||||
|
|
@ -22,8 +36,15 @@ class Db:
|
|||
self.collect_stats = False
|
||||
self.query_stats = {}
|
||||
self.db_keyvalues = {}
|
||||
self.last_query_time = time.time()
|
||||
|
||||
def __repr__(self):
|
||||
return "<Db:%s>" % self.db_path
|
||||
|
||||
def connect(self):
|
||||
if self not in opened_dbs:
|
||||
opened_dbs.append(self)
|
||||
|
||||
self.log.debug("Connecting to %s (sqlite version: %s)..." % (self.db_path, sqlite3.version))
|
||||
if not os.path.isdir(self.db_dir): # Directory not exist yet
|
||||
os.makedirs(self.db_dir)
|
||||
|
|
@ -41,16 +62,21 @@ class Db:
|
|||
|
||||
# Execute query using dbcursor
|
||||
def execute(self, query, params=None):
|
||||
self.last_query_time = time.time()
|
||||
if not self.conn:
|
||||
self.connect()
|
||||
return self.cur.execute(query, params)
|
||||
|
||||
def close(self):
|
||||
if self in opened_dbs:
|
||||
opened_dbs.remove(self)
|
||||
self.log.debug("Closing")
|
||||
if self.cur:
|
||||
self.cur.close()
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
self.conn = None
|
||||
self.cur = None
|
||||
|
||||
# Gets a cursor object to database
|
||||
# Return: Cursor class
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ class Peer(object):
|
|||
return self.connection
|
||||
|
||||
def __str__(self):
|
||||
return "Peer %-12s" % self.ip
|
||||
return "Peer:%-12s" % self.ip
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s>" % self.__str__()
|
||||
|
|
@ -126,7 +126,7 @@ class Peer(object):
|
|||
# Get a file content from peer
|
||||
def getFile(self, site, inner_path):
|
||||
# Use streamFile if client supports it
|
||||
if config.stream_downloads and self.connection and self.connection.handshake["rev"] > 310:
|
||||
if config.stream_downloads and self.connection and self.connection.handshake and self.connection.handshake["rev"] > 310:
|
||||
return self.streamFile(site, inner_path)
|
||||
|
||||
location = 0
|
||||
|
|
|
|||
|
|
@ -158,8 +158,8 @@ class Site:
|
|||
|
||||
# Download all files of the site
|
||||
@util.Noparallel(blocking=False)
|
||||
def download(self, check_size=False):
|
||||
self.log.debug("Start downloading...%s" % self.bad_files)
|
||||
def download(self, check_size=False, blind_includes=False):
|
||||
self.log.debug("Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" % (self.bad_files, check_size, blind_includes))
|
||||
gevent.spawn(self.announce)
|
||||
if check_size: # Check the size first
|
||||
valid = self.downloadContent(download_files=False) # Just download content.json files
|
||||
|
|
@ -167,10 +167,14 @@ class Site:
|
|||
return False # Cant download content.jsons or size is not fits
|
||||
|
||||
# Download everything
|
||||
found = self.downloadContent("content.json")
|
||||
self.checkModifications(0) # Download multiuser blind includes
|
||||
valid = self.downloadContent("content.json")
|
||||
|
||||
return found
|
||||
if valid and blind_includes:
|
||||
self.checkModifications(0) # Download multiuser blind includes
|
||||
|
||||
self.retryBadFiles()
|
||||
|
||||
return valid
|
||||
|
||||
# Update worker, try to find client that supports listModifications command
|
||||
def updater(self, peers_try, queried, since):
|
||||
|
|
@ -189,7 +193,9 @@ class Site:
|
|||
queried.append(peer)
|
||||
for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we
|
||||
content = self.content_manager.contents.get(inner_path)
|
||||
if not content or modified > content["modified"]: # We dont have this file or we have older
|
||||
if (not content or modified > content["modified"]) and inner_path not in self.bad_files:
|
||||
self.log.debug("New modified file from %s: %s" % (peer, inner_path))
|
||||
# We dont have this file or we have older
|
||||
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file
|
||||
gevent.spawn(self.downloadContent, inner_path) # Download the content.json + the changed files
|
||||
|
||||
|
|
@ -414,7 +420,7 @@ class Site:
|
|||
elif self.settings["serving"] is False: # Site not serving
|
||||
return False
|
||||
else: # Wait until file downloaded
|
||||
self.bad_files[inner_path] = True # Mark as bad file
|
||||
self.bad_files[inner_path] = self.bad_files.get(inner_path,0)+1 # Mark as bad file
|
||||
if not self.content_manager.contents.get("content.json"): # No content.json, download it first!
|
||||
self.log.debug("Need content.json first")
|
||||
gevent.spawn(self.announce)
|
||||
|
|
|
|||
|
|
@ -78,9 +78,12 @@ class SiteManager(object):
|
|||
if not site.settings["serving"]: # Maybe it was deleted before
|
||||
site.settings["serving"] = True
|
||||
site.saveSettings()
|
||||
if all_file: # Also download user files on first sync
|
||||
site.download(blind_includes=True)
|
||||
else:
|
||||
if all_file:
|
||||
site.download()
|
||||
|
||||
if all_file:
|
||||
site.download()
|
||||
return site
|
||||
|
||||
def delete(self, address):
|
||||
|
|
|
|||
|
|
@ -53,6 +53,10 @@ class UiRequest(object):
|
|||
# Media
|
||||
elif path.startswith("/uimedia/"):
|
||||
return self.actionUiMedia(path)
|
||||
elif "/uimedia/" in path:
|
||||
# uimedia within site dir (for chrome extension)
|
||||
path = re.sub(".*?/uimedia/", "/uimedia/", path)
|
||||
return self.actionUiMedia(path)
|
||||
elif path.startswith("/media"):
|
||||
return self.actionSiteMedia(path)
|
||||
# Websocket
|
||||
|
|
@ -258,6 +262,8 @@ class UiRequest(object):
|
|||
|
||||
# Returns if media request allowed from that referer
|
||||
def isMediaRequestAllowed(self, site_address, referer):
|
||||
if not re.sub("^http[s]{0,1}://", "", referer).startswith(self.env["HTTP_HOST"]):
|
||||
return False
|
||||
referer_path = re.sub("http[s]{0,1}://.*?/", "/", referer).replace("/media", "") # Remove site address
|
||||
return referer_path.startswith("/" + site_address)
|
||||
|
||||
|
|
|
|||
|
|
@ -530,7 +530,7 @@ class UiWebsocket(object):
|
|||
self.site.settings["size_limit"] = size_limit
|
||||
self.site.saveSettings()
|
||||
self.response(to, "Site size limit changed to %sMB" % size_limit)
|
||||
self.site.download()
|
||||
self.site.download(blind_includes=True)
|
||||
|
||||
def actionServerUpdate(self, to):
|
||||
self.cmd("updating")
|
||||
|
|
|
|||
|
|
@ -290,9 +290,9 @@ class Wrapper
|
|||
@site_error = "No peers found"
|
||||
@loading.printLine "No peers found"
|
||||
|
||||
if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").indexOf("?") == -1 # First site info and mainpage
|
||||
if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") == -1 # First site info and mainpage
|
||||
if site_info.size_limit*1.1 < site_info.next_size_limit # Need upgrade soon
|
||||
@actionConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", =>
|
||||
@displayConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", =>
|
||||
@ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) =>
|
||||
@notifications.add("size_limit", "done", res, 5000)
|
||||
return false
|
||||
|
|
@ -323,7 +323,9 @@ class Wrapper
|
|||
@loading.printLine res
|
||||
@inner_loaded = false # Inner frame not loaded, just a 404 page displayed
|
||||
if reload
|
||||
$("iframe").attr "src", $("iframe").attr("src")+"&"+(+new Date) # Reload iframe
|
||||
src = $("iframe").attr("src")
|
||||
$("iframe").attr "src", ""
|
||||
$("iframe").attr "src", src
|
||||
return false
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ a { color: black }
|
|||
|
||||
.fixbutton {
|
||||
position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999;
|
||||
text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px;
|
||||
text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px;
|
||||
}
|
||||
.fixbutton-bg {
|
||||
border-radius: 80px; background-color: rgba(180, 180, 180, 0.5); cursor: pointer;
|
||||
|
|
@ -27,7 +27,7 @@ a { color: black }
|
|||
/*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); */
|
||||
}
|
||||
.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; perspective: 1000px; line-height: 0px; padding-top: 20px }
|
||||
.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px }
|
||||
.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px }
|
||||
.fixbutton-bg:hover { background-color: #AF3BFF }
|
||||
.fixbutton-bg:active { background-color: #9E2FEA; top: 1px; transition: none }
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ a { color: black }
|
|||
|
||||
.fixbutton {
|
||||
position: absolute; right: 35px; top: 15px; width: 40px; z-index: 999;
|
||||
text-align: center; color: white; font-family: Consolas, Menlo, monospace; font-size: 25px;
|
||||
text-align: center; color: white; font-family: Consolas, Monaco, monospace; font-size: 25px;
|
||||
}
|
||||
.fixbutton-bg {
|
||||
-webkit-border-radius: 80px; -moz-border-radius: 80px; -o-border-radius: 80px; -ms-border-radius: 80px; border-radius: 80px ; background-color: rgba(180, 180, 180, 0.5); cursor: pointer;
|
||||
|
|
@ -32,7 +32,7 @@ a { color: black }
|
|||
/*box-shadow: inset 105px 260px 0px -200px rgba(0,0,0,0.1);*/ /* -webkit-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -moz-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -o-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); -ms-box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1); box-shadow: inset -75px 183px 0px -200px rgba(0,0,0,0.1) ; */
|
||||
}
|
||||
.fixbutton-text { pointer-events: none; position: absolute; z-index: 999; width: 40px; backface-visibility: hidden; -webkit-perspective: 1000px; -moz-perspective: 1000px; -o-perspective: 1000px; -ms-perspective: 1000px; perspective: 1000px ; line-height: 0px; padding-top: 20px }
|
||||
.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 48px; line-height: 32px }
|
||||
.fixbutton-burger { pointer-events: none; position: absolute; z-index: 999; width: 40px; opacity: 0; left: -20px; font-size: 40px; line-height: 0px; font-family: Verdana, sans-serif; margin-top: 17px }
|
||||
.fixbutton-bg:hover { background-color: #AF3BFF }
|
||||
.fixbutton-bg:active { background-color: #9E2FEA; top: 1px; -webkit-transition: none ; -moz-transition: none ; -o-transition: none ; -ms-transition: none ; transition: none }
|
||||
|
||||
|
|
|
|||
|
|
@ -1140,9 +1140,9 @@ jQuery.extend( jQuery.easing,
|
|||
this.loading.printLine("No peers found");
|
||||
}
|
||||
}
|
||||
if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").indexOf("?") === -1) {
|
||||
if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").replace("?wrapper=False", "").indexOf("?") === -1) {
|
||||
if (site_info.size_limit * 1.1 < site_info.next_size_limit) {
|
||||
this.actionConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) {
|
||||
this.displayConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) {
|
||||
return function() {
|
||||
_this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) {
|
||||
return _this.notifications.add("size_limit", "done", res, 5000);
|
||||
|
|
@ -1184,10 +1184,13 @@ jQuery.extend( jQuery.easing,
|
|||
}
|
||||
this.ws.cmd("siteSetLimit", [size_limit], (function(_this) {
|
||||
return function(res) {
|
||||
var src;
|
||||
_this.loading.printLine(res);
|
||||
_this.inner_loaded = false;
|
||||
if (reload) {
|
||||
return $("iframe").attr("src", $("iframe").attr("src") + "&" + (+(new Date)));
|
||||
src = $("iframe").attr("src");
|
||||
$("iframe").attr("src", "");
|
||||
return $("iframe").attr("src", src);
|
||||
}
|
||||
};
|
||||
})(this));
|
||||
|
|
@ -1234,4 +1237,4 @@ jQuery.extend( jQuery.easing,
|
|||
|
||||
window.wrapper = new Wrapper(ws_url);
|
||||
|
||||
}).call(this);
|
||||
}).call(this);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
<title>{title} - ZeroNet</title>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
|
||||
<link rel="stylesheet" href="{server_url}/uimedia/all.css?rev={rev}" />
|
||||
<link rel="stylesheet" href="/uimedia/all.css?rev={rev}" />
|
||||
{meta_tags}
|
||||
</head>
|
||||
<body style="{body_style}">
|
||||
|
|
@ -57,7 +57,7 @@ permissions = {permissions}
|
|||
show_loadingscreen = {show_loadingscreen}
|
||||
server_url = '{server_url}'
|
||||
</script>
|
||||
<script type="text/javascript" src="{server_url}/uimedia/all.js?rev={rev}"></script>
|
||||
<script type="text/javascript" src="/uimedia/all.js?rev={rev}"></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -78,7 +78,9 @@ class WorkerManager:
|
|||
return 9998 # index.html also important
|
||||
priority = task["priority"]
|
||||
if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"):
|
||||
priority += 1 # download js and css files first
|
||||
priority += 2 # boost js and css files priority
|
||||
elif task["inner_path"].endswith(".json"):
|
||||
priority += 1 # boost json files priority
|
||||
return priority - task["workers_num"] # Prefer more priority and less workers
|
||||
|
||||
# Returns the next free or less worked task
|
||||
|
|
|
|||
|
|
@ -65,6 +65,9 @@ config.parse() # Parse again to add plugin configuration options
|
|||
# Log current config
|
||||
logging.debug("Config: %s" % config)
|
||||
|
||||
# Use pure-python implementation of msgpack to save CPU
|
||||
if config.msgpack_purepython:
|
||||
os.environ["MSGPACK_PUREPYTHON"] = "True"
|
||||
|
||||
# Socks Proxy monkey patch
|
||||
if config.proxy:
|
||||
|
|
|
|||
Loading…
Reference in New Issue