diff --git a/README.md b/README.md index 21c9736f..b2ac7429 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,13 @@ Linux (Debian): - start using `python zeronet.py` +## Current limitations + - No torrent-like, file splitting big file support + - Just as anonymous as the bittorrent + - File transactions not compressed or encrypted yet + - No private sites + + ## How can I create a ZeroNet site? Shut down zeronet.py if you are running it already ``` @@ -55,6 +62,7 @@ $ zeronet.py ``` Congratulations, you are done! Now anyone can access your site using http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 + ## How can I modify a ZeroNet site? - Modify files located in data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 directory. After you done: ``` @@ -72,10 +80,13 @@ Site:13DNDk..bhC2 Successfuly published to 3 peers ``` - That's it! You successfuly signed and published your modifications. + ## If you want to help keep this project alive Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX #### Thank you! -More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/ \ No newline at end of file + +More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/ +Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet) diff --git a/src/Config.py b/src/Config.py index 11592e2f..6b2d5026 100644 --- a/src/Config.py +++ b/src/Config.py @@ -3,7 +3,7 @@ import ConfigParser class Config(object): def __init__(self): - self.version = "0.1.1" + self.version = "0.1.2" self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file @@ -53,13 +53,14 @@ class Config(object): # Config parameters parser.add_argument('--debug', help='Debug mode', action='store_true') + parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true') - parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='host') + parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') parser.add_argument('--ui_port', help='Web interface bind port', default=43110, metavar='port') parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip') parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address') - parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='host') + parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, metavar='port') parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip') diff --git a/src/Debug/Debug.py b/src/Debug/Debug.py new file mode 100644 index 00000000..3ab4c48b --- /dev/null +++ b/src/Debug/Debug.py @@ -0,0 +1,39 @@ +import sys, os, traceback + +# Non fatal exception +class Notify(Exception): + def __init__(self, message): + self.message = message + + def __str__(self): + return self.message + + +def formatException(err=None): + exc_type, exc_obj, exc_tb = sys.exc_info() + if not err: err = exc_obj.message + tb = [] + for frame in traceback.extract_tb(exc_tb): + path, line, function, text = frame + file = os.path.split(path)[1] + tb.append("%s line %s" % (file, line)) + return "%s: %s in %s" % (exc_type.__name__, err, " > ".join(tb)) + + +if __name__ == "__main__": + try: + print 1/0 + except Exception, err: + print type(err).__name__ + print "1/0 error: %s" % formatException(err) + + def loadJson(): + json.loads("Errr") + + import json + try: + loadJson() + except Exception, err: + print err + print "Json load error: %s" % formatException(err) + loadJson() diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py index d03e7fb9..60229f61 100644 --- a/src/Debug/DebugHook.py +++ b/src/Debug/DebugHook.py @@ -3,14 +3,14 @@ import gevent, sys last_error = None def handleError(*args): global last_error - if not args: # Get last error + if not args: # Called explicitly args = sys.exc_info() silent = True else: silent = False print "Error catched", args last_error = args - if not silent: sys.__excepthook__(*args) + if not silent and args[0].__name__ != "Notify": sys.__excepthook__(*args) OriginalGreenlet = gevent.Greenlet class ErrorhookedGreenlet(OriginalGreenlet): diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index e9c5926a..973b6fca 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -1,6 +1,7 @@ import os, msgpack, shutil from Site import SiteManager from cStringIO import StringIO +from Debug import Debug FILE_BUFF = 1024*512 @@ -87,7 +88,7 @@ class FileRequest: back["size"] = os.fstat(file.fileno()).st_size self.send(back) except Exception, err: - self.send({"error": "File read error: %s" % err}) + self.send({"error": "File read error: %s" % Debug.formatException(err)}) return False diff --git a/src/File/FileServer.py b/src/File/FileServer.py index 9899460d..34ce5d42 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -4,6 +4,7 @@ import zmq.green as zmq from Config import config from FileRequest import FileRequest from Site import SiteManager +from Debug import Debug class FileServer: @@ -53,7 +54,7 @@ class FileServer: else: upnpc_success = False except Exception, err: - self.log.error("Upnpc run error: %s" % err) + self.log.error("Upnpc run error: %s" % Debug.formatException(err)) upnpc_success = False if upnpc_success and self.testOpenport(port)["result"] == True: @@ -73,7 +74,7 @@ class FileServer: message = re.match('.*

(.*?)

', data, re.DOTALL).group(1) message = re.sub("<.*?>", "", message.replace("
", " ").replace(" ", " ")) # Strip http tags except Exception, err: - message = "Error: %s" % err + message = "Error: %s" % Debug.formatException(err) if "Error" in message: self.log.info("[BAD :(] Port closed: %s" % message) if port == self.port: @@ -159,7 +160,7 @@ class FileServer: self.handleRequest(req) except Exception, err: self.log.error(err) - self.socket.send(msgpack.packb({"error": "%s" % err}, use_bin_type=True)) + self.socket.send(msgpack.packb({"error": "%s" % Debug.formatException(err)}, use_bin_type=True)) if config.debug: # Raise exception import sys sys.excepthook(*sys.exc_info()) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 43c31659..6a4bb6f9 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -2,6 +2,7 @@ import os, logging, gevent, time, msgpack import zmq.green as zmq from cStringIO import StringIO from Config import config +from Debug import Debug context = zmq.Context() @@ -40,23 +41,29 @@ class Peer: # Send a command to peer def sendCmd(self, cmd, params = {}): if not self.socket: self.connect() - self.log.debug("sendCmd: %s" % cmd) - try: - self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True)) - response = msgpack.unpackb(self.socket.recv()) - if "error" in response: - self.log.debug("%s %s error: %s" % (cmd, params, response["error"])) + for retry in range(1,5): + if config.debug_socket: self.log.debug("sendCmd: %s" % cmd) + try: + self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True)) + if config.debug_socket: self.log.debug("Sent command: %s" % cmd) + response = msgpack.unpackb(self.socket.recv()) + if config.debug_socket: self.log.debug("Got response to: %s" % cmd) + if "error" in response: + self.log.debug("%s error: %s" % (cmd, response["error"])) + self.onConnectionError() + else: # Successful request, reset connection error num + self.connection_error = 0 + return response + except Exception, err: self.onConnectionError() - else: # Successful request, reset connection error num - self.connection_error = 0 - return response - except Exception, err: - self.onConnectionError() - self.log.error("%s" % err) - self.socket.close() - time.sleep(1) - self.connect() - return None + self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry)) + self.socket.close() + time.sleep(1*retry) + self.connect() + if type(err).__name__ == "Notify" and err.message == "Worker stopped": # Greenlet kill by worker + self.log.debug("Peer worker got killed, aborting cmd: %s" % cmd) + break + return None # Failed after 4 retry # Get a file content from peer @@ -97,7 +104,7 @@ class Peer: # On connection error def onConnectionError(self): self.connection_error += 1 - if self.connection_error > 5: # Dead peer + if self.connection_error >= 5: # Dead peer self.remove() diff --git a/src/Site/Site.py b/src/Site/Site.py index 163862e1..6443b9db 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -6,6 +6,7 @@ from Config import config from Peer import Peer from Worker import WorkerManager from Crypt import CryptHash +from Debug import Debug import SiteManager class Site: @@ -53,7 +54,7 @@ class Site: try: new_content = json.load(open(content_path)) except Exception, err: - self.log.error("Content.json load error: %s" % err) + self.log.error("Content.json load error: %s" % Debug.formatException(err)) return None else: return None # Content.json not exits @@ -69,7 +70,7 @@ class Site: if old_sha1 != new_sha1: changed.append(inner_path) self.content = new_content except Exception, err: - self.log.error("Content.json parse error: %s" % err) + self.log.error("Content.json parse error: %s" % Debug.formatException(err)) return None # Content.json parse error # Add to bad files if not init: @@ -114,7 +115,7 @@ class Site: # Start downloading site @util.Noparallel(blocking=False) def download(self): - self.log.debug("Start downloading...") + self.log.debug("Start downloading...%s" % self.bad_files) self.announce() found = self.needFile("content.json", update=self.bad_files.get("content.json")) if not found: return False # Could not download content.json @@ -165,7 +166,7 @@ class Site: "peer": (config.ip_external, config.fileserver_port) }) except Exception, err: - result = {"exception": err} + result = {"exception": Debug.formatException(err)} if result and "ok" in result: published += 1 @@ -231,24 +232,22 @@ class Site: tracker.poll_once() tracker.announce(info_hash=hashlib.sha1(self.address).hexdigest(), num_want=50) back = tracker.poll_once() - except Exception, err: - self.log.error("Tracker error: %s" % err) - continue - if back: # Tracker announce success peers = back["response"]["peers"] - added = 0 - for peer in peers: - if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself) - continue - if self.addPeer(peer["addr"], peer["port"]): added += 1 - if added: - self.worker_manager.onPeers() - self.updateWebsocket(peers_added=added) - self.log.debug("Found %s peers, new: %s" % (len(peers), added)) - break # Successful announcing, break the list - else: - self.log.error("Tracker bad response, trying next in list...") # Failed to announce, go to next + except Exception, err: + self.log.error("Tracker error: %s" % Debug.formatException(err)) time.sleep(1) + continue + + added = 0 + for peer in peers: + if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself) + continue + if self.addPeer(peer["addr"], peer["port"]): added += 1 + if added: + self.worker_manager.onPeers() + self.updateWebsocket(peers_added=added) + self.log.debug("Found %s peers, new: %s" % (len(peers), added)) + break # Successful announcing, break the list else: pass # TODO: http tracker support @@ -342,7 +341,7 @@ class Site: return CryptBitcoin.verify(sign_content, self.address, sign) except Exception, err: - self.log.error("Verify sign error: %s" % err) + self.log.error("Verify sign error: %s" % Debug.formatException(err)) return False else: # Check using sha1 hash diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index c870e157..611173c6 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -245,6 +245,7 @@ class UiRequest: # Just raise an error to get console def actionConsole(self): + sites = self.server.sites raise Exception("Here is your console") diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 61f32a8f..e14a72bc 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -6,6 +6,7 @@ from lib.geventwebsocket.handler import WebSocketHandler from Ui import UiRequest from Site import SiteManager from Config import config +from Debug import Debug # Skip websocket handler if not necessary class UiWSGIHandler(WSGIHandler): @@ -48,19 +49,6 @@ class UiServer: return self.ui_request.route(path) - # Send a message to all connected client - def sendMessage(self, message): - sent = 0 - for ws in self.websockets: - try: - ws.send(message) - sent += 1 - except Exception, err: - self.log.error("addMessage error: %s" % err) - self.server.websockets.remove(ws) - return sent - - # Reload the UiRequest class to prevent restarts in debug mode def reload(self): import imp diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index e5736287..68d3ce77 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -1,6 +1,7 @@ import json, gevent, time, sys, hashlib from Config import config from Site import SiteManager +from Debug import Debug class UiWebsocket: def __init__(self, ws, site, server): @@ -36,7 +37,7 @@ class UiWebsocket: if config.debug: # Allow websocket errors to appear on /Debug import sys sys.modules["src.main"].DebugHook.handleError() - self.log.error("WebSocket error: %s" % err) + self.log.error("WebSocket error: %s" % Debug.formatException(err)) return "Bye." @@ -70,7 +71,7 @@ class UiWebsocket: if cb: # Callback after client responsed self.waiting_cb[message["id"]] = cb except Exception, err: - self.log.debug("Websocket send error: %s" % err) + self.log.debug("Websocket send error: %s" % Debug.formatException(err)) # Handle incoming messages @@ -152,7 +153,7 @@ class UiWebsocket: # Server variables def actionServerInfo(self, to, params): ret = { - "ip_external": config.ip_external, + "ip_external": bool(config.ip_external), "platform": sys.platform, "fileserver_ip": config.fileserver_ip, "fileserver_port": config.fileserver_port, diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py index 797bc35c..bbe9486c 100644 --- a/src/Worker/Worker.py +++ b/src/Worker/Worker.py @@ -1,5 +1,6 @@ import gevent, time, logging, shutil, os from Peer import Peer +from Debug import Debug class Worker: def __init__(self, manager, peer): @@ -66,6 +67,11 @@ class Worker: self.running = True self.thread = gevent.spawn(self.downloader) + + # Force stop the worker def stop(self): + self.manager.log.debug("%s: Force stopping, thread: %s" % (self.key, self.thread)) self.running = False + if self.thread: + self.thread.kill(exception=Debug.Notify("Worker stopped")) self.manager.removeWorker(self) diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index a2ea1dac..dd2ba7b3 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -16,7 +16,12 @@ class WorkerManager: # Check expired tasks def checkTasks(self): while 1: - time.sleep(15) # Check every 30 sec + time.sleep(15) # Check every 15 sec + + # Clean up workers + if not self.tasks and self.workers: # No task but workers still running + for worker in self.workers.values(): worker.stop() + if not self.tasks: continue tasks = self.tasks[:] # Copy it so removing elements wont cause any problem for task in tasks: @@ -40,6 +45,7 @@ class WorkerManager: + # Tasks sorted by this def taskSorter(self, task): if task["inner_path"] == "content.json": return 9999 # Content.json always prority @@ -96,8 +102,9 @@ class WorkerManager: # Ends and remove a worker def removeWorker(self, worker): worker.running = False - if worker.key in self.workers: del(self.workers[worker.key]) - self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS)) + if worker.key in self.workers: + del(self.workers[worker.key]) + self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS)) # Create new task and return asyncresult diff --git a/src/main.py b/src/main.py index ab8fc25f..791e952c 100644 --- a/src/main.py +++ b/src/main.py @@ -19,8 +19,14 @@ if config.action == "main": else: logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main +# Console logger console_log = logging.StreamHandler() -console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S")) +if config.action == "main": # Add time if main action + console_log.setFormatter(logging.Formatter('[%(asctime)s] %(name)s %(message)s', "%H:%M:%S")) +else: + console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S")) + + logging.getLogger('').addHandler(console_log) # Add console logger logging.getLogger('').name = "-" # Remove root prefix