Save significant amount of memory by remove unused msgpack unpackers, Log unhandled exceptions, Connectionserver checker error bugfix

src/Connection/Connection.py

@@ -21,7 +21,7 @@ class Connection:
 
 		self.server = server
 		self.log = logging.getLogger(str(self))
-		self.unpacker = msgpack.Unpacker() # Stream incoming socket messages here
+		self.unpacker = None # Stream incoming socket messages here
 		self.req_id = 0 # Last request id
 		self.handshake = {} # Handshake info got from peer
 		self.connected = False
@@ -133,8 +133,8 @@ class Connection:
 			self.connected = True
 			self.event_connected.set(self.protocol) # Mark handshake as done
 
-			unpacker = self.unpacker
-			unpacker.feed(firstchar) # Feed the first char we already requested
+			self.unpacker = msgpack.Unpacker()
+			self.unpacker.feed(firstchar) # Feed the first char we already requested
 			try:
 				while True:
 					buff = sock.recv(16*1024)
@@ -142,8 +142,11 @@ class Connection:
 					self.last_recv_time = time.time()
 					self.incomplete_buff_recv += 1
 					self.bytes_recv += len(buff)
-					unpacker.feed(buff)
-					for message in unpacker:
+					if not self.unpacker: 
+						self.log.debug("Unpacker created")
+						self.unpacker = msgpack.Unpacker()
+					self.unpacker.feed(buff)
+					for message in self.unpacker:
 						self.incomplete_buff_recv = 0
 						self.handleMessage(message)
 					message = None

src/Connection/ConnectionServer.py

@@ -112,8 +112,13 @@ class ConnectionServer:
 			for connection in self.connections[:]: # Make a copy
 				idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
 
+				if connection.unpacker and idle > 30: # Delete the unpacker if not needed
+					del connection.unpacker
+					connection.unpacker = None
+					connection.log.debug("Unpacker deleted")
+
 				if idle > 60*60: # Wake up after 1h
-					connection.log.debug("[Cleanup] After wakeup: %s" % connection.read_bytes(1024))
+					connection.log.debug("[Cleanup] After wakeup, idle: %s" % idle)
 					connection.close()
 
 				elif idle > 20*60 and connection.last_send_time < time.time()-10: # Idle more than 20 min and we not send request in last 10 sec
@@ -125,7 +130,7 @@ class ConnectionServer:
 							connection.close()
 
 				elif idle > 10 and connection.incomplete_buff_recv > 0: # Incompelte data with more than 10 sec idle
-					connection.log.debug("[Cleanup] Connection buff stalled, content: %s" % connection.read_bytes(1024))
+					connection.log.debug("[Cleanup] Connection buff stalled")
 					connection.close()
 
 				elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10: # Sent command and no response in 10 sec

src/Debug/DebugHook.py

@@ -19,7 +19,9 @@ def handleError(*args):
 
 # Ignore notify errors
 def handleErrorNotify(*args):
-	if args[0].__name__ != "Notify": sys.__excepthook__(*args)
+	if args[0].__name__ != "Notify": 
+		logging.exception("Unhandled exception")
+		sys.__excepthook__(*args)
 
 
 OriginalGreenlet = gevent.Greenlet

src/File/FileRequest.py

@@ -55,7 +55,7 @@ class FileRequest:
 		buff = StringIO(params["body"])
 		valid = site.content_manager.verifyFile(params["inner_path"], buff)
 		if valid == True: # Valid and changed
-			self.log.debug("Update for %s looks valid, saving..." % params["inner_path"])
+			self.log.info("Update for %s looks valid, saving..." % params["inner_path"])
 			buff.seek(0)
 			file = open(site.getPath(params["inner_path"]), "wb")
 			shutil.copyfileobj(buff, file) # Write buff to disk