new documents with examples, raise exception on connection error, close connection if no handshake for 60 sec, log unhandled exceptions, calc object size using guppy, typo fix

README.md

@@ -89,7 +89,7 @@ $ zeronet.py
 Congratulations, you're finished! Now anyone can access your site using
 `http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2`
 
-Next steps: [ZeroNet Developer Documentation](https://github.com/HelloZeroNet/ZeroNet/wiki/ZeroNet-Developer-Documentation)
+Next steps: [ZeroNet Developer Documentation](http://zeronet.readthedocs.org/en/latest/site_development/debug_mode/)
 
 
 ## How can I modify a ZeroNet site?

src/Connection/Connection.py

@@ -264,6 +264,8 @@ class Connection:
 	def close(self):
 		if self.closed: return False # Already closed
 		self.closed = True
+		self.event_connected.set(False)
+		
 		if config.debug_socket: self.log.debug("Closing connection, waiting_requests: %s, buff: %s..." % (len(self.waiting_requests), self.incomplete_buff_recv))
 		for request in self.waiting_requests.values(): # Mark pending requests failed
 			request.set(False)

src/Connection/ConnectionServer.py

@@ -62,17 +62,22 @@ class ConnectionServer:
 	def getConnection(self, ip=None, port=None, peer_id=None):
 		if peer_id and peer_id in self.peer_ids: # Find connection by peer id
 			connection = self.peer_ids.get(peer_id)
-			if not connection.connected: connection.event_connected.get() # Wait for connection
+			if not connection.connected: 
+				succ = connection.event_connected.get() # Wait for connection
+				if not succ: raise Exception("Connection event return error")
 			return connection
 		if ip in self.ips: # Find connection by ip
 			connection = self.ips[ip]
-			if not connection.connected: connection.event_connected.get() # Wait for connection
+			if not connection.connected: 
+				succ = connection.event_connected.get() # Wait for connection
+				if not succ: raise Exception("Connection event return error")
 			return connection
-
 		# Recover from connection pool
 		for connection in self.connections:
 			if connection.ip == ip: 
-				if not connection.connected: connection.event_connected.get() # Wait for connection
+				if not connection.connected: 
+					succ = connection.event_connected.get() # Wait for connection
+					if not succ: raise Exception("Connection event return error")
 				return connection
 
 		# No connection found
@@ -80,7 +85,9 @@ class ConnectionServer:
 			connection = Connection(self, ip, port)
 			self.ips[ip] = connection
 			self.connections.append(connection)
-			connection.connect()
+			succ = connection.connect()
+			if not succ:
+				raise Exception("Connection event return error")
 		except Exception, err:
 			self.log.debug("%s Connect error: %s" % (ip, Debug.formatException(err)))
 			connection.close()
@@ -103,26 +110,32 @@ class ConnectionServer:
 		while self.running:
 			time.sleep(60) # Sleep 1 min
 			for connection in self.connections[:]: # Make a copy
-				if connection.protocol == "zeromq": continue # No stat on ZeroMQ sockets
-				idle = time.time() - max(connection.last_recv_time, connection.start_time)
+				idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
 
 				if idle > 60*60: # Wake up after 1h
+					connection.log.debug("[Cleanup] After wakeup: %s" % connection.read_bytes(1024))
 					connection.close()
 
 				elif idle > 20*60 and connection.last_send_time < time.time()-10: # Idle more than 20 min and we not send request in last 10 sec
-					if connection.protocol == "?": connection.close() # Got no handshake response, close it
+					if connection.protocol == "zeromq":
+						if idle > 50*60 and not connection.ping(): # Only ping every 50 sec
+							connection.close()
 					else: 
 						if not connection.ping(): # send ping request
 							connection.close()
 
 				elif idle > 10 and connection.incomplete_buff_recv > 0: # Incompelte data with more than 10 sec idle
-					connection.log.debug("[Cleanup] Connection buff stalled, content: %s" % connection.u.read_bytes(1024))
+					connection.log.debug("[Cleanup] Connection buff stalled, content: %s" % connection.read_bytes(1024))
 					connection.close()
 
 				elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10: # Sent command and no response in 10 sec
 					connection.log.debug("[Cleanup] Command %s timeout: %s" % (connection.last_cmd, time.time() - connection.last_send_time))
 					connection.close()
 
+				elif idle > 60 and connection.protocol == "?": # No connection after 1 min
+					connection.log.debug("[Cleanup] Connect timeout: %s" % idle)
+					connection.close()
+
 
 
 	def zmqServer(self):

src/Debug/DebugHook.py

@@ -1,4 +1,4 @@
-import gevent, sys
+import gevent, sys, logging
 from Config import config
 
 last_error = None
@@ -13,6 +13,7 @@ def handleError(*args):
 		silent = False
 	if args[0].__name__ != "Notify": last_error = args
 	if not silent and args[0].__name__ != "Notify": 
+		logging.exception("Unhandled exception")
 		sys.__excepthook__(*args)
 
 

src/Debug/DebugMedia.py

@@ -58,7 +58,7 @@ def merge(merged_path):
 					parts.append(out)
 				else:
 					error = out
-					logging.error("%s Compile error %s:" % (file_path, error))
+					logging.error("%s Compile error: %s" % (file_path, error))
 					parts.append("alert('%s compile error: %s');" % (file_path, re.escape(error).replace("\n", "\\n").replace(r"\\n", r"\n") ) )
 			else: # Not changed use the old_part
 				parts.append(old_parts[file_path])
@@ -78,4 +78,4 @@ if __name__ == "__main__":
 	logging.getLogger().setLevel(logging.DEBUG)
 	os.chdir("..") 
 	config.coffeescript_compiler = r'type "%s" | tools\coffee-node\bin\node.exe tools\coffee-node\bin\coffee --no-header -s -p'
-	merge("data/1TaLk3zM7ZRskJvrh3ZNCDVGXvkJusPKQ/js/all.js")
+	merge("data/12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH/js/all.js")

src/Peer/Peer.py

@@ -40,8 +40,7 @@ class Peer:
 		try:
 			self.connection = self.connection_server.getConnection(self.ip, self.port)
 		except Exception, err:
-			self.log.debug("Getting connection error: %s" % Debug.formatException(err))
-			self.onConnectionError()
+			self.log.debug("Getting connection error: %s (connection_error: %s, hash_failed: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed))
 			
 	def __str__(self):
 		return "Peer %-12s" % self.ip
@@ -58,7 +57,9 @@ class Peer:
 	def request(self, cmd, params = {}):
 		if not self.connection or self.connection.closed: 
 			self.connect()
-			if not self.connection: return None # Connection failed
+			if not self.connection: 
+				self.onConnectionError()
+				return None # Connection failed
 
 		#if cmd != "ping" and self.last_response and time.time() - self.last_response > 20*60: # If last response if older than 20 minute, ping first to see if still alive
 		#	if not self.ping(): return None

src/Site/Site.py

@@ -51,6 +51,15 @@ class Site:
 		self.addEventListeners()
 
 
+
+	def __str__(self):
+		return "Site %s" % self.address_short
+
+
+	def __repr__(self):
+		return "<%s>" % self.__str__()
+
+
 	# Load site settings from data/sites.json
 	def loadSettings(self):
 		sites_settings = json.load(open("data/sites.json"))

src/Ui/UiRequest.py

@@ -291,11 +291,29 @@ class UiRequest:
 			back.append("<td>%s</td>" % formatted)
 		return "<tr>%s</tr>" % "".join(back)
 
+
+	def getObjSize(self, obj, hpy = None):
+		if hpy:
+			return float(hpy.iso(obj).domisize)/1024
+		else:
+			return 0
+
+
+
 	def actionStats(self):
 		import gc, sys
+		hpy = None
+		if self.get.get("size") == "1": # Calc obj size
+			try:
+				import guppy
+				hpy = guppy.hpy()
+			except:
+				pass
 		self.sendHeader()
 		s = time.time()
 		main = sys.modules["src.main"]
+
+		# Style
 		yield """
 		<style>
 		 * { font-family: monospace }
@@ -313,7 +331,7 @@ class UiRequest:
 			yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
 			yield "Open files: %s | " % len(process.open_files())
 			yield "Sockets: %s" % len(process.connections())
-			yield "<br>"
+			yield " | Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a><br>"
 		except Exception, err:
 			pass
 
@@ -344,33 +362,40 @@ class UiRequest:
 		objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
 		yield "<br>Greenlets (%s):<br>" % len(objs)
 		for obj in objs:
-			yield " - %sbyte: %s<br>" % (sys.getsizeof(obj), cgi.escape(repr(obj)))
+			yield " - %.3fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
 
 
 		from Worker import Worker
 		objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
 		yield "<br>Workers (%s):<br>" % len(objs)
 		for obj in objs:
-			yield " - %sbyte: %s<br>" % (sys.getsizeof(obj), cgi.escape(repr(obj)))
+			yield " - %.3fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
 
 
 		from Connection import Connection
 		objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
 		yield "<br>Connections (%s):<br>" % len(objs)
 		for obj in objs:
-			yield " - %sbyte: %s<br>" % (sys.getsizeof(obj), cgi.escape(repr(obj)))
+			yield " - %.3fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
+
+
+		from Site import Site
+		objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
+		yield "<br>Sites (%s):<br>" % len(objs)
+		for obj in objs:
+			yield " - %.3fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
 
 
 		objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
 		yield "<br>Loggers (%s):<br>" % len(objs)
 		for obj in objs:
-			yield " - %sbyte: %s<br>" % (sys.getsizeof(obj), cgi.escape(repr(obj.name)))
+			yield " - %.3fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj.name)))
 
 
 		objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
 		yield "<br>UiRequest (%s):<br>" % len(objs)
 		for obj in objs:
-			yield " - %sbyte: %s<br>" % (sys.getsizeof(obj), cgi.escape(repr(obj)))
+			yield " - %.3fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
 
 		yield "Done in %.3f" % (time.time()-s)
 

src/Ui/UiWebsocket.py

@@ -172,6 +172,7 @@ class UiWebsocket:
 
 		settings = site.settings.copy()
 		del settings["wrapper_key"] # Dont expose wrapper key
+		del settings["auth_key"] # Dont send auth key twice
 
 		ret = {
 			"auth_key": self.site.settings["auth_key"], # Obsolete, will be removed

src/util/UpnpPunch.py

@@ -178,7 +178,7 @@ def open_port(port=15441, desc="UpnpPunch"):
 
 	local_ips = [_get_local_ip()]
 	try:
-		local_ips += socket.gethostbyname_ex('')[2] # Get ip by '' hostrname not supported on all platform
+		local_ips += socket.gethostbyname_ex('')[2] # Get ip by '' hostname not supported on all platform
 	except:
 		pass