limitations and irc to readme, version 0.1.2, socket debugging option, Notify exceptions support, better error logging, retry on socket error, dont expose external ip to websocket api, kill workers if no task, log time to console

README.md

@@ -41,6 +41,13 @@ Linux (Debian):
  - start using `python zeronet.py`
 
 
+## Current limitations
+ - No torrent-like, file splitting big file support
+ - Just as anonymous as the bittorrent
+ - File transactions not compressed or encrypted yet
+ - No private sites
+
+
 ## How can I create a ZeroNet site?
 Shut down zeronet.py if you are running it already
 ```
@@ -55,6 +62,7 @@ $ zeronet.py
 ```
 Congratulations, you are done! Now anyone can access your site using http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
 
+
 ## How can I modify a ZeroNet site?
 - Modify files located in data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 directory. After you done:
 ```
@@ -72,10 +80,13 @@ Site:13DNDk..bhC2 Successfuly published to 3 peers
 ```
 - That's it! You successfuly signed and published your modifications.
 
+
 ## If you want to help keep this project alive
 
 Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX
 
 #### Thank you!
 
-More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
+
+More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
+Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet)

src/Config.py

@@ -3,7 +3,7 @@ import ConfigParser
 
 class Config(object):
 	def __init__(self):
-		self.version = "0.1.1"
+		self.version = "0.1.2"
 		self.parser = self.createArguments()
 		argv = sys.argv[:] # Copy command line arguments
 		argv = self.parseConfig(argv) # Add arguments from config file
@@ -53,13 +53,14 @@ class Config(object):
 
 		# Config parameters
 		parser.add_argument('--debug', 			help='Debug mode', action='store_true')
+		parser.add_argument('--debug_socket', 	help='Debug socket connections', action='store_true')
 
-		parser.add_argument('--ui_ip', 			help='Web interface bind address', default="127.0.0.1", metavar='host')
+		parser.add_argument('--ui_ip', 			help='Web interface bind address', default="127.0.0.1", metavar='ip')
 		parser.add_argument('--ui_port', 		help='Web interface bind port', default=43110, metavar='port')
 		parser.add_argument('--ui_restrict',	help='Restrict web access', default=False, metavar='ip')
 		parser.add_argument('--homepage',		help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
 
-		parser.add_argument('--fileserver_ip', 	help='FileServer bind address', default="*", metavar='host')
+		parser.add_argument('--fileserver_ip', 	help='FileServer bind address', default="*", metavar='ip')
 		parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, metavar='port')
 
 		parser.add_argument('--ip_external',	help='External ip (tested on start if None)', metavar='ip')

src/Debug/Debug.py

@@ -0,0 +1,39 @@
+import sys, os, traceback
+
+# Non fatal exception
+class Notify(Exception):
+	def __init__(self, message):
+		self.message = message
+
+	def __str__(self):
+		return self.message
+
+
+def formatException(err=None):
+	exc_type, exc_obj, exc_tb = sys.exc_info()
+	if not err: err = exc_obj.message
+	tb = []
+	for frame in traceback.extract_tb(exc_tb):
+		path, line, function, text = frame
+		file = os.path.split(path)[1]
+		tb.append("%s line %s" % (file, line))
+	return "%s: %s in %s" % (exc_type.__name__, err, " > ".join(tb))
+
+
+if __name__ == "__main__":
+	try:
+		print 1/0
+	except Exception, err:
+		print type(err).__name__
+		print "1/0 error: %s" % formatException(err)
+	
+	def loadJson():
+		json.loads("Errr")
+
+	import json
+	try:
+		loadJson()
+	except Exception, err:
+		print err
+		print "Json load error: %s" % formatException(err)
+	loadJson()

src/Debug/DebugHook.py

@@ -3,14 +3,14 @@ import gevent, sys
 last_error = None
 def handleError(*args):
 	global last_error
-	if not args: # Get last error
+	if not args: # Called explicitly
 		args = sys.exc_info()
 		silent = True
 	else:
 		silent = False
 	print "Error catched", args
 	last_error = args
-	if not silent: sys.__excepthook__(*args)
+	if not silent and args[0].__name__ != "Notify": sys.__excepthook__(*args)
 
 OriginalGreenlet = gevent.Greenlet
 class ErrorhookedGreenlet(OriginalGreenlet):

src/File/FileRequest.py

@@ -1,6 +1,7 @@
 import os, msgpack, shutil
 from Site import SiteManager
 from cStringIO import StringIO
+from Debug import Debug
 
 FILE_BUFF = 1024*512
 
@@ -87,7 +88,7 @@ class FileRequest:
 			back["size"] = os.fstat(file.fileno()).st_size
 			self.send(back)
 		except Exception, err:
-			self.send({"error": "File read error: %s" % err})
+			self.send({"error": "File read error: %s" % Debug.formatException(err)})
 			return False
 
 

src/File/FileServer.py

@@ -4,6 +4,7 @@ import zmq.green as zmq
 from Config import config
 from FileRequest import FileRequest
 from Site import SiteManager
+from Debug import Debug
 
 
 class FileServer:
@@ -53,7 +54,7 @@ class FileServer:
 				else:
 					upnpc_success = False
 			except Exception, err:
-				self.log.error("Upnpc run error: %s" % err)
+				self.log.error("Upnpc run error: %s" % Debug.formatException(err))
 				upnpc_success = False
 
 			if upnpc_success and self.testOpenport(port)["result"] == True:
@@ -73,7 +74,7 @@ class FileServer:
 			message = re.match('.*<p style="padding-left:15px">(.*?)</p>', data, re.DOTALL).group(1)
 			message = re.sub("<.*?>", "", message.replace("<br>", " ").replace("&nbsp;", " ")) # Strip http tags
 		except Exception, err:
-			message = "Error: %s" % err
+			message = "Error: %s" % Debug.formatException(err)
 		if "Error" in message:
 			self.log.info("[BAD :(] Port closed: %s" % message)
 			if port == self.port: 
@@ -159,7 +160,7 @@ class FileServer:
 				self.handleRequest(req)
 			except Exception, err:
 				self.log.error(err)
-				self.socket.send(msgpack.packb({"error": "%s" % err}, use_bin_type=True))
+				self.socket.send(msgpack.packb({"error": "%s" % Debug.formatException(err)}, use_bin_type=True))
 				if config.debug: # Raise exception
 					import sys
 					sys.excepthook(*sys.exc_info())

src/Peer/Peer.py

@@ -2,6 +2,7 @@ import os, logging, gevent, time, msgpack
 import zmq.green as zmq
 from cStringIO import StringIO
 from Config import config
+from Debug import Debug
 
 context = zmq.Context()
 
@@ -40,23 +41,29 @@ class Peer:
 	# Send a command to peer
 	def sendCmd(self, cmd, params = {}):
 		if not self.socket: self.connect()
-		self.log.debug("sendCmd: %s" % cmd)
+		for retry in range(1,5):
-		try:
+			if config.debug_socket: self.log.debug("sendCmd: %s" % cmd)
-			self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True))
+			try:
-			response = msgpack.unpackb(self.socket.recv())
+				self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True))
-			if "error" in response:
+				if config.debug_socket: self.log.debug("Sent command: %s" % cmd)
-				self.log.debug("%s %s error: %s" % (cmd, params, response["error"]))
+				response = msgpack.unpackb(self.socket.recv())
+				if config.debug_socket: self.log.debug("Got response to: %s" % cmd)
+				if "error" in response:
+					self.log.debug("%s error: %s" % (cmd, response["error"]))
+					self.onConnectionError()
+				else: # Successful request, reset connection error num
+					self.connection_error = 0
+				return response
+			except Exception, err:
 				self.onConnectionError()
-			else: # Successful request, reset connection error num
+				self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry))
-				self.connection_error = 0
+				self.socket.close()
-			return response
+				time.sleep(1*retry)
-		except Exception, err:
+				self.connect()
-			self.onConnectionError()
+				if type(err).__name__ == "Notify" and err.message == "Worker stopped": # Greenlet kill by worker
-			self.log.error("%s" % err)
+					self.log.debug("Peer worker got killed, aborting cmd: %s" % cmd)
-			self.socket.close()
+					break
-			time.sleep(1)
+		return None # Failed after 4 retry
-			self.connect()
-			return None
 
 
 	# Get a file content from peer
@@ -97,7 +104,7 @@ class Peer:
 	# On connection error
 	def onConnectionError(self):
 		self.connection_error += 1
-		if self.connection_error > 5: # Dead peer
+		if self.connection_error >= 5: # Dead peer
 			self.remove()
 
 

src/Site/Site.py

@@ -6,6 +6,7 @@ from Config import config
 from Peer import Peer
 from Worker import WorkerManager
 from Crypt import CryptHash
+from Debug import Debug
 import SiteManager
 
 class Site:
@@ -53,7 +54,7 @@ class Site:
 			try:
 				new_content = json.load(open(content_path))
 			except Exception, err:
-				self.log.error("Content.json load error: %s" % err)
+				self.log.error("Content.json load error: %s" % Debug.formatException(err))
 				return None
 		else:
 			return None # Content.json not exits
@@ -69,7 +70,7 @@ class Site:
 				if old_sha1 != new_sha1: changed.append(inner_path)
 			self.content = new_content
 		except Exception, err:
-			self.log.error("Content.json parse error: %s" % err)
+			self.log.error("Content.json parse error: %s" % Debug.formatException(err))
 			return None # Content.json parse error
 		# Add to bad files
 		if not init:
@@ -114,7 +115,7 @@ class Site:
 	# Start downloading site
 	@util.Noparallel(blocking=False)
 	def download(self):
-		self.log.debug("Start downloading...")
+		self.log.debug("Start downloading...%s" % self.bad_files)
 		self.announce()
 		found = self.needFile("content.json", update=self.bad_files.get("content.json"))
 		if not found: return False # Could not download content.json
@@ -165,7 +166,7 @@ class Site:
 						"peer": (config.ip_external, config.fileserver_port)
 					})
 			except Exception, err:
-				result = {"exception": err}
+				result = {"exception": Debug.formatException(err)}
 
 			if result and "ok" in result:
 				published += 1
@@ -231,24 +232,22 @@ class Site:
 					tracker.poll_once()
 					tracker.announce(info_hash=hashlib.sha1(self.address).hexdigest(), num_want=50)
 					back = tracker.poll_once()
-				except Exception, err:
-					self.log.error("Tracker error: %s" % err)
-					continue
-				if back: # Tracker announce success
 					peers = back["response"]["peers"]
-					added = 0
+				except Exception, err:
-					for peer in peers:
+					self.log.error("Tracker error: %s" % Debug.formatException(err))
-						if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself)
-							continue
-						if self.addPeer(peer["addr"], peer["port"]): added += 1
-					if added:
-						self.worker_manager.onPeers()
-						self.updateWebsocket(peers_added=added)
-					self.log.debug("Found %s peers, new: %s" % (len(peers), added))
-					break # Successful announcing, break the list
-				else:
-					self.log.error("Tracker bad response, trying next in list...") # Failed to announce, go to next
 					time.sleep(1)
+					continue
+			
+				added = 0
+				for peer in peers:
+					if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself)
+						continue
+					if self.addPeer(peer["addr"], peer["port"]): added += 1
+				if added:
+					self.worker_manager.onPeers()
+					self.updateWebsocket(peers_added=added)
+				self.log.debug("Found %s peers, new: %s" % (len(peers), added))
+				break # Successful announcing, break the list					
 			else:
 				pass # TODO: http tracker support
 
@@ -342,7 +341,7 @@ class Site:
 
 				return CryptBitcoin.verify(sign_content, self.address, sign)
 			except Exception, err:
-				self.log.error("Verify sign error: %s" % err)
+				self.log.error("Verify sign error: %s" % Debug.formatException(err))
 				return False
 
 		else: # Check using sha1 hash

src/Ui/UiRequest.py

@@ -245,6 +245,7 @@ class UiRequest:
 
 	# Just raise an error to get console
 	def actionConsole(self):
+		sites = self.server.sites
 		raise Exception("Here is your console")
 
 

src/Ui/UiServer.py

@@ -6,6 +6,7 @@ from lib.geventwebsocket.handler import WebSocketHandler
 from Ui import UiRequest
 from Site import SiteManager
 from Config import config
+from Debug import Debug
 
 # Skip websocket handler if not necessary
 class UiWSGIHandler(WSGIHandler):
@@ -48,19 +49,6 @@ class UiServer:
 		return self.ui_request.route(path)
 
 
-	# Send a message to all connected client
-	def sendMessage(self, message):
-		sent = 0
-		for ws in self.websockets:
-			try:
-				ws.send(message)
-				sent += 1
-			except Exception, err:
-				self.log.error("addMessage error: %s" % err)
-				self.server.websockets.remove(ws)
-		return sent
-
-
 	# Reload the UiRequest class to prevent restarts in debug mode
 	def reload(self):
 		import imp

src/Ui/UiWebsocket.py

@@ -1,6 +1,7 @@
 import json, gevent, time, sys, hashlib
 from Config import config
 from Site import SiteManager
+from Debug import Debug
 
 class UiWebsocket:
 	def __init__(self, ws, site, server):
@@ -36,7 +37,7 @@ class UiWebsocket:
 					if config.debug: # Allow websocket errors to appear on /Debug 
 						import sys
 						sys.modules["src.main"].DebugHook.handleError() 
-					self.log.error("WebSocket error: %s" % err)
+					self.log.error("WebSocket error: %s" % Debug.formatException(err))
 				return "Bye."
 
 
@@ -70,7 +71,7 @@ class UiWebsocket:
 			if cb: # Callback after client responsed
 				self.waiting_cb[message["id"]] = cb
 		except Exception, err:
-			self.log.debug("Websocket send error: %s" % err)
+			self.log.debug("Websocket send error: %s" % Debug.formatException(err))
 
 
 	# Handle incoming messages
@@ -152,7 +153,7 @@ class UiWebsocket:
 	# Server variables
 	def actionServerInfo(self, to, params):
 		ret = {
-			"ip_external": config.ip_external,
+			"ip_external": bool(config.ip_external),
 			"platform": sys.platform,
 			"fileserver_ip": config.fileserver_ip,
 			"fileserver_port": config.fileserver_port,

src/Worker/Worker.py

@@ -1,5 +1,6 @@
 import gevent, time, logging, shutil, os
 from Peer import Peer
+from Debug import Debug
 
 class Worker:
 	def __init__(self, manager, peer):
@@ -66,6 +67,11 @@ class Worker:
 		self.running = True
 		self.thread = gevent.spawn(self.downloader)
 
+
+	# Force stop the worker
 	def stop(self):
+		self.manager.log.debug("%s: Force stopping, thread: %s" % (self.key, self.thread))
 		self.running = False
+		if self.thread:
+			self.thread.kill(exception=Debug.Notify("Worker stopped"))
 		self.manager.removeWorker(self)

src/Worker/WorkerManager.py

@@ -16,7 +16,12 @@ class WorkerManager:
 	# Check expired tasks
 	def checkTasks(self):
 		while 1:
-			time.sleep(15) # Check every 30 sec
+			time.sleep(15) # Check every 15 sec
+
+			# Clean up workers
+			if not self.tasks and self.workers: # No task but workers still running
+				for worker in self.workers.values(): worker.stop()
+
 			if not self.tasks: continue
 			tasks = self.tasks[:] # Copy it so removing elements wont cause any problem
 			for task in tasks:
@@ -40,6 +45,7 @@ class WorkerManager:
 
 
 
+
 	# Tasks sorted by this
 	def taskSorter(self, task):
 		if task["inner_path"] == "content.json": return 9999 # Content.json always prority
@@ -96,8 +102,9 @@ class WorkerManager:
 	# Ends and remove a worker
 	def removeWorker(self, worker):
 		worker.running = False
-		if worker.key in self.workers: del(self.workers[worker.key])
+		if worker.key in self.workers: 
-		self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS))
+			del(self.workers[worker.key])
+			self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS))
 
 
 	# Create new task and return asyncresult

src/main.py

@@ -19,8 +19,14 @@ if config.action == "main":
 else:
 	logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main
 
+# Console logger
 console_log = logging.StreamHandler()
-console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S"))
+if config.action == "main": # Add time if main action
+	console_log.setFormatter(logging.Formatter('[%(asctime)s] %(name)s %(message)s', "%H:%M:%S"))
+else:
+	console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S"))
+
+
 logging.getLogger('').addHandler(console_log) # Add console logger
 logging.getLogger('').name = "-" # Remove root prefix