From b37e309eda8591dcd126d169e2ab4621bfdc3326 Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Sat, 17 Jan 2015 18:50:56 +0100 Subject: [PATCH] limitations and irc to readme, version 0.1.2, socket debugging option, Notify exceptions support, better error logging, retry on socket error, dont expose external ip to websocket api, kill workers if no task, log time to console --- README.md | 13 +++++++++++- src/Config.py | 7 ++++--- src/Debug/Debug.py | 39 +++++++++++++++++++++++++++++++++++ src/Debug/DebugHook.py | 4 ++-- src/File/FileRequest.py | 3 ++- src/File/FileServer.py | 7 ++++--- src/Peer/Peer.py | 41 ++++++++++++++++++++++--------------- src/Site/Site.py | 41 ++++++++++++++++++------------------- src/Ui/UiRequest.py | 1 + src/Ui/UiServer.py | 14 +------------ src/Ui/UiWebsocket.py | 7 ++++--- src/Worker/Worker.py | 6 ++++++ src/Worker/WorkerManager.py | 13 +++++++++--- src/main.py | 8 +++++++- 14 files changed, 136 insertions(+), 68 deletions(-) create mode 100644 src/Debug/Debug.py diff --git a/README.md b/README.md index 21c9736f..b2ac7429 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,13 @@ Linux (Debian): - start using `python zeronet.py` +## Current limitations + - No torrent-like, file splitting big file support + - Just as anonymous as the bittorrent + - File transactions not compressed or encrypted yet + - No private sites + + ## How can I create a ZeroNet site? Shut down zeronet.py if you are running it already ``` @@ -55,6 +62,7 @@ $ zeronet.py ``` Congratulations, you are done! Now anyone can access your site using http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 + ## How can I modify a ZeroNet site? - Modify files located in data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 directory. After you done: ``` @@ -72,10 +80,13 @@ Site:13DNDk..bhC2 Successfuly published to 3 peers ``` - That's it! You successfuly signed and published your modifications. + ## If you want to help keep this project alive Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX #### Thank you! -More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/ \ No newline at end of file + +More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/ +Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) diff --git a/src/Config.py b/src/Config.py index 11592e2f..6b2d5026 100644 --- a/src/Config.py +++ b/src/Config.py @@ -3,7 +3,7 @@ import ConfigParser class Config(object): def __init__(self): - self.version = "0.1.1" + self.version = "0.1.2" self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file @@ -53,13 +53,14 @@ class Config(object): # Config parameters parser.add_argument('--debug', help='Debug mode', action='store_true') + parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true') - parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='host') + parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') parser.add_argument('--ui_port', help='Web interface bind port', default=43110, metavar='port') parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip') parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address') - parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='host') + parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, metavar='port') parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip') diff --git a/src/Debug/Debug.py b/src/Debug/Debug.py new file mode 100644 index 00000000..3ab4c48b --- /dev/null +++ b/src/Debug/Debug.py @@ -0,0 +1,39 @@ +import sys, os, traceback + +# Non fatal exception +class Notify(Exception): + def __init__(self, message): + self.message = message + + def __str__(self): + return self.message + + +def formatException(err=None): + exc_type, exc_obj, exc_tb = sys.exc_info() + if not err: err = exc_obj.message + tb = [] + for frame in traceback.extract_tb(exc_tb): + path, line, function, text = frame + file = os.path.split(path)[1] + tb.append("%s line %s" % (file, line)) + return "%s: %s in %s" % (exc_type.__name__, err, " > ".join(tb)) + + +if __name__ == "__main__": + try: + print 1/0 + except Exception, err: + print type(err).__name__ + print "1/0 error: %s" % formatException(err) + + def loadJson(): + json.loads("Errr") + + import json + try: + loadJson() + except Exception, err: + print err + print "Json load error: %s" % formatException(err) + loadJson() diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py index d03e7fb9..60229f61 100644 --- a/src/Debug/DebugHook.py +++ b/src/Debug/DebugHook.py @@ -3,14 +3,14 @@ import gevent, sys last_error = None def handleError(*args): global last_error - if not args: # Get last error + if not args: # Called explicitly args = sys.exc_info() silent = True else: silent = False print "Error catched", args last_error = args - if not silent: sys.__excepthook__(*args) + if not silent and args[0].__name__ != "Notify": sys.__excepthook__(*args) OriginalGreenlet = gevent.Greenlet class ErrorhookedGreenlet(OriginalGreenlet): diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index e9c5926a..973b6fca 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -1,6 +1,7 @@ import os, msgpack, shutil from Site import SiteManager from cStringIO import StringIO +from Debug import Debug FILE_BUFF = 1024*512 @@ -87,7 +88,7 @@ class FileRequest: back["size"] = os.fstat(file.fileno()).st_size self.send(back) except Exception, err: - self.send({"error": "File read error: %s" % err}) + self.send({"error": "File read error: %s" % Debug.formatException(err)}) return False diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 9899460d..34ce5d42 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -4,6 +4,7 @@ import zmq.green as zmq from Config import config from FileRequest import FileRequest from Site import SiteManager +from Debug import Debug class FileServer: @@ -53,7 +54,7 @@ class FileServer: else: upnpc_success = False except Exception, err: - self.log.error("Upnpc run error: %s" % err) + self.log.error("Upnpc run error: %s" % Debug.formatException(err)) upnpc_success = False if upnpc_success and self.testOpenport(port)["result"] == True: @@ -73,7 +74,7 @@ class FileServer: message = re.match('.*

(.*?)

', data, re.DOTALL).group(1) message = re.sub("<.*?>", "", message.replace("
", " ").replace(" ", " ")) # Strip http tags except Exception, err: - message = "Error: %s" % err + message = "Error: %s" % Debug.formatException(err) if "Error" in message: self.log.info("[BAD :(] Port closed: %s" % message) if port == self.port: @@ -159,7 +160,7 @@ class FileServer: self.handleRequest(req) except Exception, err: self.log.error(err) - self.socket.send(msgpack.packb({"error": "%s" % err}, use_bin_type=True)) + self.socket.send(msgpack.packb({"error": "%s" % Debug.formatException(err)}, use_bin_type=True)) if config.debug: # Raise exception import sys sys.excepthook(*sys.exc_info()) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 43c31659..6a4bb6f9 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -2,6 +2,7 @@ import os, logging, gevent, time, msgpack import zmq.green as zmq from cStringIO import StringIO from Config import config +from Debug import Debug context = zmq.Context() @@ -40,23 +41,29 @@ class Peer: # Send a command to peer def sendCmd(self, cmd, params = {}): if not self.socket: self.connect() - self.log.debug("sendCmd: %s" % cmd) - try: - self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True)) - response = msgpack.unpackb(self.socket.recv()) - if "error" in response: - self.log.debug("%s %s error: %s" % (cmd, params, response["error"])) + for retry in range(1,5): + if config.debug_socket: self.log.debug("sendCmd: %s" % cmd) + try: + self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True)) + if config.debug_socket: self.log.debug("Sent command: %s" % cmd) + response = msgpack.unpackb(self.socket.recv()) + if config.debug_socket: self.log.debug("Got response to: %s" % cmd) + if "error" in response: + self.log.debug("%s error: %s" % (cmd, response["error"])) + self.onConnectionError() + else: # Successful request, reset connection error num + self.connection_error = 0 + return response + except Exception, err: self.onConnectionError() - else: # Successful request, reset connection error num - self.connection_error = 0 - return response - except Exception, err: - self.onConnectionError() - self.log.error("%s" % err) - self.socket.close() - time.sleep(1) - self.connect() - return None + self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry)) + self.socket.close() + time.sleep(1*retry) + self.connect() + if type(err).__name__ == "Notify" and err.message == "Worker stopped": # Greenlet kill by worker + self.log.debug("Peer worker got killed, aborting cmd: %s" % cmd) + break + return None # Failed after 4 retry # Get a file content from peer @@ -97,7 +104,7 @@ class Peer: # On connection error def onConnectionError(self): self.connection_error += 1 - if self.connection_error > 5: # Dead peer + if self.connection_error >= 5: # Dead peer self.remove() diff --git a/src/Site/Site.py b/src/Site/Site.py index 163862e1..6443b9db 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -6,6 +6,7 @@ from Config import config from Peer import Peer from Worker import WorkerManager from Crypt import CryptHash +from Debug import Debug import SiteManager class Site: @@ -53,7 +54,7 @@ class Site: try: new_content = json.load(open(content_path)) except Exception, err: - self.log.error("Content.json load error: %s" % err) + self.log.error("Content.json load error: %s" % Debug.formatException(err)) return None else: return None # Content.json not exits @@ -69,7 +70,7 @@ class Site: if old_sha1 != new_sha1: changed.append(inner_path) self.content = new_content except Exception, err: - self.log.error("Content.json parse error: %s" % err) + self.log.error("Content.json parse error: %s" % Debug.formatException(err)) return None # Content.json parse error # Add to bad files if not init: @@ -114,7 +115,7 @@ class Site: # Start downloading site @util.Noparallel(blocking=False) def download(self): - self.log.debug("Start downloading...") + self.log.debug("Start downloading...%s" % self.bad_files) self.announce() found = self.needFile("content.json", update=self.bad_files.get("content.json")) if not found: return False # Could not download content.json @@ -165,7 +166,7 @@ class Site: "peer": (config.ip_external, config.fileserver_port) }) except Exception, err: - result = {"exception": err} + result = {"exception": Debug.formatException(err)} if result and "ok" in result: published += 1 @@ -231,24 +232,22 @@ class Site: tracker.poll_once() tracker.announce(info_hash=hashlib.sha1(self.address).hexdigest(), num_want=50) back = tracker.poll_once() - except Exception, err: - self.log.error("Tracker error: %s" % err) - continue - if back: # Tracker announce success peers = back["response"]["peers"] - added = 0 - for peer in peers: - if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself) - continue - if self.addPeer(peer["addr"], peer["port"]): added += 1 - if added: - self.worker_manager.onPeers() - self.updateWebsocket(peers_added=added) - self.log.debug("Found %s peers, new: %s" % (len(peers), added)) - break # Successful announcing, break the list - else: - self.log.error("Tracker bad response, trying next in list...") # Failed to announce, go to next + except Exception, err: + self.log.error("Tracker error: %s" % Debug.formatException(err)) time.sleep(1) + continue + + added = 0 + for peer in peers: + if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself) + continue + if self.addPeer(peer["addr"], peer["port"]): added += 1 + if added: + self.worker_manager.onPeers() + self.updateWebsocket(peers_added=added) + self.log.debug("Found %s peers, new: %s" % (len(peers), added)) + break # Successful announcing, break the list else: pass # TODO: http tracker support @@ -342,7 +341,7 @@ class Site: return CryptBitcoin.verify(sign_content, self.address, sign) except Exception, err: - self.log.error("Verify sign error: %s" % err) + self.log.error("Verify sign error: %s" % Debug.formatException(err)) return False else: # Check using sha1 hash diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index c870e157..611173c6 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -245,6 +245,7 @@ class UiRequest: # Just raise an error to get console def actionConsole(self): + sites = self.server.sites raise Exception("Here is your console") diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 61f32a8f..e14a72bc 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -6,6 +6,7 @@ from lib.geventwebsocket.handler import WebSocketHandler from Ui import UiRequest from Site import SiteManager from Config import config +from Debug import Debug # Skip websocket handler if not necessary class UiWSGIHandler(WSGIHandler): @@ -48,19 +49,6 @@ class UiServer: return self.ui_request.route(path) - # Send a message to all connected client - def sendMessage(self, message): - sent = 0 - for ws in self.websockets: - try: - ws.send(message) - sent += 1 - except Exception, err: - self.log.error("addMessage error: %s" % err) - self.server.websockets.remove(ws) - return sent - - # Reload the UiRequest class to prevent restarts in debug mode def reload(self): import imp diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index e5736287..68d3ce77 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -1,6 +1,7 @@ import json, gevent, time, sys, hashlib from Config import config from Site import SiteManager +from Debug import Debug class UiWebsocket: def __init__(self, ws, site, server): @@ -36,7 +37,7 @@ class UiWebsocket: if config.debug: # Allow websocket errors to appear on /Debug import sys sys.modules["src.main"].DebugHook.handleError() - self.log.error("WebSocket error: %s" % err) + self.log.error("WebSocket error: %s" % Debug.formatException(err)) return "Bye." @@ -70,7 +71,7 @@ class UiWebsocket: if cb: # Callback after client responsed self.waiting_cb[message["id"]] = cb except Exception, err: - self.log.debug("Websocket send error: %s" % err) + self.log.debug("Websocket send error: %s" % Debug.formatException(err)) # Handle incoming messages @@ -152,7 +153,7 @@ class UiWebsocket: # Server variables def actionServerInfo(self, to, params): ret = { - "ip_external": config.ip_external, + "ip_external": bool(config.ip_external), "platform": sys.platform, "fileserver_ip": config.fileserver_ip, "fileserver_port": config.fileserver_port, diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py index 797bc35c..bbe9486c 100644 --- a/src/Worker/Worker.py +++ b/src/Worker/Worker.py @@ -1,5 +1,6 @@ import gevent, time, logging, shutil, os from Peer import Peer +from Debug import Debug class Worker: def __init__(self, manager, peer): @@ -66,6 +67,11 @@ class Worker: self.running = True self.thread = gevent.spawn(self.downloader) + + # Force stop the worker def stop(self): + self.manager.log.debug("%s: Force stopping, thread: %s" % (self.key, self.thread)) self.running = False + if self.thread: + self.thread.kill(exception=Debug.Notify("Worker stopped")) self.manager.removeWorker(self) diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index a2ea1dac..dd2ba7b3 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -16,7 +16,12 @@ class WorkerManager: # Check expired tasks def checkTasks(self): while 1: - time.sleep(15) # Check every 30 sec + time.sleep(15) # Check every 15 sec + + # Clean up workers + if not self.tasks and self.workers: # No task but workers still running + for worker in self.workers.values(): worker.stop() + if not self.tasks: continue tasks = self.tasks[:] # Copy it so removing elements wont cause any problem for task in tasks: @@ -40,6 +45,7 @@ class WorkerManager: + # Tasks sorted by this def taskSorter(self, task): if task["inner_path"] == "content.json": return 9999 # Content.json always prority @@ -96,8 +102,9 @@ class WorkerManager: # Ends and remove a worker def removeWorker(self, worker): worker.running = False - if worker.key in self.workers: del(self.workers[worker.key]) - self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS)) + if worker.key in self.workers: + del(self.workers[worker.key]) + self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS)) # Create new task and return asyncresult diff --git a/src/main.py b/src/main.py index ab8fc25f..791e952c 100644 --- a/src/main.py +++ b/src/main.py @@ -19,8 +19,14 @@ if config.action == "main": else: logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main +# Console logger console_log = logging.StreamHandler() -console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S")) +if config.action == "main": # Add time if main action + console_log.setFormatter(logging.Formatter('[%(asctime)s] %(name)s %(message)s', "%H:%M:%S")) +else: + console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S")) + + logging.getLogger('').addHandler(console_log) # Add console logger logging.getLogger('').name = "-" # Remove root prefix