From 531bf68dddac2b084f6dd815e1b7f86b8e22b19a Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Fri, 20 Feb 2015 01:37:12 +0100 Subject: [PATCH] Version 0.2.3, One click updater from github, Clean FileServer and UiServer shutdown, Count UiServer http connections to clean close, serverUpdate WrapperAPI command, randomize peers before work start and publish, switched to upnpc-shared it has better virustotal reputation --- src/Config.py | 4 +-- src/File/FileServer.py | 20 ++++++++++---- src/Site/Site.py | 3 -- src/Ui/UiServer.py | 36 +++++++++++++++++++++--- src/Ui/UiWebsocket.py | 11 ++++++++ src/Ui/media/Wrapper.coffee | 5 +++- src/Ui/media/all.js | 16 +++++++---- src/Ui/media/lib/ZeroWebsocket.coffee | 8 +++--- src/Worker/WorkerManager.py | 8 ++++-- src/main.py | 3 +- update.py | 39 ++++++++++++++++++++++++++ zeronet.py | 40 +++++++++++++++++++++------ 12 files changed, 157 insertions(+), 36 deletions(-) create mode 100644 update.py diff --git a/src/Config.py b/src/Config.py index ee9f00f2..2b0b0538 100644 --- a/src/Config.py +++ b/src/Config.py @@ -3,7 +3,7 @@ import ConfigParser class Config(object): def __init__(self): - self.version = "0.2.2" + self.version = "0.2.3" self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file @@ -19,7 +19,7 @@ class Config(object): def createArguments(self): # Platform specific if sys.platform.startswith("win"): - upnpc = "tools\\upnpc\\upnpc-static.exe" + upnpc = "tools\\upnpc\\upnpc-shared.exe" coffeescript = "type %s | tools\\coffee\\coffee.cmd" else: upnpc = None diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 4f2c2b6c..5ae5b732 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -18,6 +18,7 @@ class FileServer: else: self.port_opened = None # Is file server opened on router self.sites = SiteManager.list() + self.running = True # Handle request to fileserver @@ -168,17 +169,26 @@ class FileServer: if check_sites: # Open port, Update sites, Check files integrity gevent.spawn(self.checkSites) - gevent.spawn(self.announceSites) - gevent.spawn(self.wakeupWatcher) + thread_announce_sites = gevent.spawn(self.announceSites) + thread_wakeup_watcher = gevent.spawn(self.wakeupWatcher) - while True: + while self.running: try: ret = {} req = msgpack.unpackb(socket.recv()) self.handleRequest(req) except Exception, err: self.log.error(err) - self.socket.send(msgpack.packb({"error": "%s" % Debug.formatException(err)}, use_bin_type=True)) + if self.running: self.socket.send(msgpack.packb({"error": "%s" % Debug.formatException(err)}, use_bin_type=True)) if config.debug: # Raise exception import sys - sys.excepthook(*sys.exc_info()) + sys.modules["src.main"].DebugHook.handleError() + thread_wakeup_watcher.kill(exception=Debug.Notify("Stopping FileServer")) + thread_announce_sites.kill(exception=Debug.Notify("Stopping FileServer")) + self.log.debug("Stopped.") + + + def stop(self): + self.running = False + self.socket.close() + diff --git a/src/Site/Site.py b/src/Site/Site.py index 21cf7105..590ed856 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -194,9 +194,6 @@ class Site: self.log.info("[OK] %s: %s" % (peer.key, result["ok"])) else: self.log.info("[ERROR] %s: %s" % (peer.key, result)) - - - # Update content.json on peers diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index cc15d8e2..0592a019 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -11,20 +11,25 @@ from Debug import Debug # Skip websocket handler if not necessary class UiWSGIHandler(WSGIHandler): def __init__(self, *args, **kwargs): + self.server = args[2] super(UiWSGIHandler, self).__init__(*args, **kwargs) - self.ws_handler = WebSocketHandler(*args, **kwargs) + self.args = args + self.kwargs = kwargs def run_application(self): + self.server.sockets[self.client_address] = self.socket if "HTTP_UPGRADE" in self.environ: # Websocket request - self.ws_handler.__dict__ = self.__dict__ # Match class variables - self.ws_handler.run_application() + ws_handler = WebSocketHandler(*self.args, **self.kwargs) + ws_handler.__dict__ = self.__dict__ # Match class variables + ws_handler.run_application() else: # Standard HTTP request #print self.application.__class__.__name__ try: return super(UiWSGIHandler, self).run_application() except Exception, err: logging.debug("UiWSGIHandler error: %s" % err) + del self.server.sockets[self.client_address] class UiServer: @@ -89,4 +94,27 @@ class UiServer: browser = webbrowser.get(config.open_browser) browser.open("http://%s:%s" % (config.ui_ip, config.ui_port), new=2) - WSGIServer((self.ip, self.port), handler, handler_class=UiWSGIHandler, log=self.log).serve_forever() + self.server = WSGIServer((self.ip, self.port), handler, handler_class=UiWSGIHandler, log=self.log) + self.server.sockets = {} + self.server.serve_forever() + self.log.debug("Stopped.") + + def stop(self): + # Close WS sockets + for client in self.server.clients.values(): + client.ws.close() + # Close http sockets + sock_closed = 0 + for sock in self.server.sockets.values(): + try: + sock._sock.close() + sock.close() + sock_closed += 1 + except Exception, err: + pass + self.log.debug("Socket closed: %s" % sock_closed) + + self.server.socket.close() + self.server.stop() + time.sleep(1) + diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index 38614a53..dec12fb5 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -121,6 +121,8 @@ class UiWebsocket: func = self.actionSiteSetLimit elif cmd == "channelJoinAllsite" and "ADMIN" in permissions: func = self.actionChannelJoinAllsite + elif cmd == "serverUpdate" and "ADMIN" in permissions: + func = self.actionServerUpdate # Unknown command else: self.response(req["id"], "Unknown command: %s" % cmd) @@ -361,3 +363,12 @@ class UiWebsocket: self.site.saveSettings() self.response(to, "Site size limit changed to %sMB" % size_limit) self.site.download() + + + def actionServerUpdate(self, to): + import sys + self.cmd("updating") + sys.modules["src.main"].update_after_shutdown = True + sys.modules["src.main"].file_server.stop() + sys.modules["src.main"].ui_server.stop() + diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee index f52dcfe0..2b87f6c1 100644 --- a/src/Ui/media/Wrapper.coffee +++ b/src/Ui/media/Wrapper.coffee @@ -46,6 +46,9 @@ class Wrapper @sendInner message # Pass to inner frame if message.params.address == window.address # Current page @setSiteInfo message.params + else if cmd == "updating" # Close connection + @ws.ws.close() + @ws.onCloseWebsocket(null, 4000) else @sendInner message # Pass message to inner frame @@ -166,7 +169,7 @@ class Wrapper @wrapperWsInited = false setTimeout (=> # Wait a bit, maybe its page closing @sendInner {"cmd": "wrapperClosedWebsocket"} # Send to inner frame - if e.code == 1000 # Server error please reload page + if e and e.code == 1000 and e.wasClean == false # Server error please reload page @ws_error = @notifications.add("connection", "error", "UiServer Websocket error, please reload the page.") else if not @ws_error @ws_error = @notifications.add("connection", "error", "Connection with UiServer Websocket was lost. Reconnecting...") diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index 0c4ae7eb..401459dd 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -120,17 +120,20 @@ } }; - ZeroWebsocket.prototype.onCloseWebsocket = function(e) { + ZeroWebsocket.prototype.onCloseWebsocket = function(e, reconnect) { + if (reconnect == null) { + reconnect = 10000; + } this.log("Closed", e); - if (e.code === 1000) { - this.log("Server error, please reload the page"); + if (e && e.code === 1000 && e.wasClean === false) { + this.log("Server error, please reload the page", e.wasClean); } else { setTimeout(((function(_this) { return function() { _this.log("Reconnecting..."); return _this.connect(); }; - })(this)), 10000); + })(this)), reconnect); } if (this.onClose != null) { return this.onClose(e); @@ -780,6 +783,9 @@ jQuery.extend( jQuery.easing, if (message.params.address === window.address) { return this.setSiteInfo(message.params); } + } else if (cmd === "updating") { + this.ws.ws.close(); + return this.ws.onCloseWebsocket(null, 4000); } else { return this.sendInner(message); } @@ -947,7 +953,7 @@ jQuery.extend( jQuery.easing, _this.sendInner({ "cmd": "wrapperClosedWebsocket" }); - if (e.code === 1000) { + if (e && e.code === 1000 && e.wasClean === false) { return _this.ws_error = _this.notifications.add("connection", "error", "UiServer Websocket error, please reload the page."); } else if (!_this.ws_error) { return _this.ws_error = _this.notifications.add("connection", "error", "Connection with UiServer Websocket was lost. Reconnecting..."); diff --git a/src/Ui/media/lib/ZeroWebsocket.coffee b/src/Ui/media/lib/ZeroWebsocket.coffee index daa5228f..eebafa0a 100644 --- a/src/Ui/media/lib/ZeroWebsocket.coffee +++ b/src/Ui/media/lib/ZeroWebsocket.coffee @@ -66,15 +66,15 @@ class ZeroWebsocket if @onError? then @onError(e) - onCloseWebsocket: (e) => + onCloseWebsocket: (e, reconnect=10000) => @log "Closed", e - if e.code == 1000 - @log "Server error, please reload the page" + if e and e.code == 1000 and e.wasClean == false + @log "Server error, please reload the page", e.wasClean else # Connection error setTimeout (=> @log "Reconnecting..." @connect() - ), 10000 + ), reconnect if @onClose? then @onClose(e) diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index 7c970639..e5bf3915 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -1,5 +1,5 @@ from Worker import Worker -import gevent, time, logging +import gevent, time, logging, random MAX_WORKERS = 10 @@ -87,10 +87,12 @@ class WorkerManager: def startWorkers(self, peers=None): if len(self.workers) >= MAX_WORKERS and not peers: return False # Workers number already maxed if not self.tasks: return False # No task for workers - for key, peer in self.site.peers.iteritems(): # One worker for every peer + peers = self.site.peers.values() + random.shuffle(peers) + for peer in peers: # One worker for every peer if peers and peer not in peers: continue # If peers definied and peer not valid worker = self.addWorker(peer) - if worker: self.log.debug("Added worker: %s, workers: %s/%s" % (key, len(self.workers), MAX_WORKERS)) + if worker: self.log.debug("Added worker: %s, workers: %s/%s" % (peer.key, len(self.workers), MAX_WORKERS)) # Stop all worker diff --git a/src/main.py b/src/main.py index 03deb90e..f1385eb2 100644 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,5 @@ import os, sys +update_after_shutdown = False sys.path.insert(0, os.path.dirname(__file__)) # Imports relative to main.py # Create necessary files and dirs @@ -43,7 +44,6 @@ else: import gevent import time - logging.debug("Starting... %s" % config) # Starts here when running zeronet.py @@ -56,6 +56,7 @@ def start(): # Start serving UiServer and PeerServer def main(): + global ui_server, file_server from File import FileServer from Ui import UiServer logging.info("Creating UiServer....") diff --git a/update.py b/update.py new file mode 100644 index 00000000..e27cda45 --- /dev/null +++ b/update.py @@ -0,0 +1,39 @@ +from gevent import monkey; monkey.patch_all() +import urllib, zipfile, os, ssl, httplib, socket +import cStringIO as StringIO + +def update(): + # Gevent https bug workaround (https://github.com/gevent/gevent/issues/477) + reload(socket) + reload(httplib) + reload(ssl) + + print "Downloading.", + file = urllib.urlopen("https://github.com/HelloZeroNet/ZeroNet/archive/master.zip") + data = StringIO.StringIO() + while True: + buff = file.read(1024*16) + if not buff: break + data.write(buff) + print ".", + + print "Extracting...", + zip = zipfile.ZipFile(data) + for inner_path in zip.namelist(): + print ".", + dest_path = inner_path.replace("ZeroNet-master/", "") + if not dest_path: continue + + dest_dir = os.path.dirname(dest_path) + if dest_dir and not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + + if dest_dir != dest_path.strip("/"): + data = zip.read(inner_path) + open(dest_path, 'wb').write(data) + + print "Done." + + +if __name__ == "__main__": + update() \ No newline at end of file diff --git a/zeronet.py b/zeronet.py index f8a83fa5..e8156209 100644 --- a/zeronet.py +++ b/zeronet.py @@ -1,13 +1,37 @@ #!/usr/bin/env python def main(): - try: - from src import main - main.start() - except Exception, err: # Prevent closing - import traceback - traceback.print_exc() - raw_input("-- Error happened, press enter to close --") + try: + from src import main + main.start() + if main.update_after_shutdown: # Updater + import update, sys, os, gc + # Update + update.update() + + # Close log files + logger = sys.modules["src.main"].logging.getLogger() + + for handler in logger.handlers[:]: + handler.flush() + handler.close() + logger.removeHandler(handler) + + except Exception, err: # Prevent closing + import traceback + traceback.print_exc() + raw_input("-- Error happened, press enter to close --") + + if main.update_after_shutdown: # Updater + # Restart + gc.collect() # Garbage collect + print "Restarting..." + args = sys.argv[:] + args.insert(0, sys.executable) + if sys.platform == 'win32': + args = ['"%s"' % arg for arg in args] + os.execv(sys.executable, args) + print "Bye." if __name__ == '__main__': - main() + main()