From dc791a31abf660fbf05e49dd54661af9346d46c6 Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Thu, 25 Jun 2015 20:09:41 +0200 Subject: [PATCH] rev259, Protection against connection flood, Fix site size limit error dialog, Convert ConnectionServer and ContentManager to PEP8 format --- src/Config.py | 2 +- src/Connection/ConnectionServer.py | 373 +++++++++++------------------ src/Content/ContentManager.py | 235 ++++++++++-------- src/Peer/Peer.py | 2 +- src/Ui/media/Wrapper.coffee | 11 +- src/Ui/media/all.js | 17 +- 6 files changed, 301 insertions(+), 339 deletions(-) diff --git a/src/Config.py b/src/Config.py index a8063ddd..2a53e71c 100644 --- a/src/Config.py +++ b/src/Config.py @@ -4,7 +4,7 @@ import ConfigParser class Config(object): def __init__(self): self.version = "0.3.1" - self.rev = 247 + self.rev = 259 self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index e862c959..84cd28ce 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -1,8 +1,14 @@ +import logging +import random +import string +import time +import sys + +import gevent +import msgpack from gevent.server import StreamServer from gevent.pool import Pool -import socket, os, logging, random, string, time, sys -import gevent, msgpack -import cStringIO as StringIO + from Debug import Debug from Connection import Connection from Config import config @@ -10,250 +16,163 @@ from Crypt import CryptConnection class ConnectionServer: - def __init__(self, ip=None, port=None, request_handler=None): - self.ip = ip - self.port = port - self.last_connection_id = 1 # Connection id incrementer - self.log = logging.getLogger("ConnServer") - self.port_opened = None - - self.connections = [] # Connections - self.ips = {} # Connection by ip - self.peer_ids = {} # Connections by peer_ids + def __init__(self, ip=None, port=None, request_handler=None): + self.ip = ip + self.port = port + self.last_connection_id = 1 # Connection id incrementer + self.log = logging.getLogger("ConnServer") + self.port_opened = None - self.running = True - self.thread_checker = gevent.spawn(self.checkConnections) + self.connections = [] # Connections + self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood + self.ips = {} # Connection by ip + self.peer_ids = {} # Connections by peer_ids - self.bytes_recv = 0 - self.bytes_sent = 0 + self.running = True + self.thread_checker = gevent.spawn(self.checkConnections) - self.peer_id = "-ZN0"+config.version.replace(".", "")+"-"+''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12)) # Bittorrent style peerid + self.bytes_recv = 0 + self.bytes_sent = 0 - # Check msgpack version - if msgpack.version[0] == 0 and msgpack.version[1] < 4: - self.log.error("Error: Too old msgpack version: %s (>0.4.0 required), please update using `sudo pip install msgpack-python --upgrade`" % str(msgpack.version)) - import sys - sys.exit(0) + # Bittorrent style peerid + self.peer_id = "-ZN0%s-%s" % ( + config.version.replace(".", ""), + ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12)) + ) - if port: # Listen server on a port - self.pool = Pool(1000) # do not accept more than 1000 connections - self.stream_server = StreamServer((ip.replace("*", ""), port), self.handleIncomingConnection, spawn=self.pool, backlog=100) - if request_handler: self.handleRequest = request_handler + # Check msgpack version + if msgpack.version[0] == 0 and msgpack.version[1] < 4: + self.log.error( + "Error: Too old msgpack version: %s (>0.4.0 required), please update using `sudo pip install msgpack-python --upgrade`" % + str(msgpack.version) + ) + sys.exit(0) - CryptConnection.manager.loadCerts() + if port: # Listen server on a port + self.pool = Pool(1000) # do not accept more than 1000 connections + self.stream_server = StreamServer((ip.replace("*", ""), port), self.handleIncomingConnection, spawn=self.pool, backlog=100) + if request_handler: + self.handleRequest = request_handler + CryptConnection.manager.loadCerts() - def start(self): - self.running = True - self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % (self.ip, self.port, ".".join(map(str, msgpack.version)), CryptConnection.manager.crypt_supported ) ) - try: - self.stream_server.serve_forever() # Start normal connection server - except Exception, err: - self.log.info("StreamServer bind error, must be running already: %s" % err) + def start(self): + self.running = True + self.log.debug("Binding to: %s:%s, (msgpack: %s), supported crypt: %s" % ( + self.ip, self.port, + ".".join(map(str, msgpack.version)), CryptConnection.manager.crypt_supported) + ) + try: + self.stream_server.serve_forever() # Start normal connection server + except Exception, err: + self.log.info("StreamServer bind error, must be running already: %s" % err) + def stop(self): + self.running = False + self.stream_server.stop() - def stop(self): - self.running = False - self.stream_server.stop() + def handleIncomingConnection(self, sock, addr): + ip, port = addr + # Connection flood protection + if ip in self.ip_incoming: + self.ip_incoming[ip] += 1 + if self.ip_incoming[ip] > 3: # Allow 3 in 1 minute from same ip + self.log.debug("Connection flood detected from %s" % ip) + time.sleep(30) + sock.close() + return False + else: + self.ip_incoming[ip] = 0 - def handleIncomingConnection(self, sock, addr): - ip, port = addr - connection = Connection(self, ip, port, sock) - self.connections.append(connection) - self.ips[ip] = connection - connection.handleIncomingConnection(sock) + connection = Connection(self, ip, port, sock) + self.connections.append(connection) + self.ips[ip] = connection + connection.handleIncomingConnection(sock) + def getConnection(self, ip=None, port=None, peer_id=None, create=True): + if peer_id and peer_id in self.peer_ids: # Find connection by peer id + connection = self.peer_ids.get(peer_id) + if not connection.connected and create: + succ = connection.event_connected.get() # Wait for connection + if not succ: + raise Exception("Connection event return error") + return connection + # Find connection by ip + if ip in self.ips: + connection = self.ips[ip] + if not connection.connected and create: + succ = connection.event_connected.get() # Wait for connection + if not succ: + raise Exception("Connection event return error") + return connection + # Recover from connection pool + for connection in self.connections: + if connection.ip == ip: + if not connection.connected and create: + succ = connection.event_connected.get() # Wait for connection + if not succ: + raise Exception("Connection event return error") + return connection + # No connection found + if create: # Allow to create new connection if not found + if port == 0: + raise Exception("This peer is not connectable") + try: + connection = Connection(self, ip, port) + self.ips[ip] = connection + self.connections.append(connection) + succ = connection.connect() + if not succ: + connection.close() + raise Exception("Connection event return error") - def getConnection(self, ip=None, port=None, peer_id=None, create=True): - if peer_id and peer_id in self.peer_ids: # Find connection by peer id - connection = self.peer_ids.get(peer_id) - if not connection.connected and create: - succ = connection.event_connected.get() # Wait for connection - if not succ: raise Exception("Connection event return error") - return connection - # Find connection by ip - if ip in self.ips: - connection = self.ips[ip] - if not connection.connected and create: - succ = connection.event_connected.get() # Wait for connection - if not succ: raise Exception("Connection event return error") - return connection - # Recover from connection pool - for connection in self.connections: - if connection.ip == ip: - if not connection.connected and create: - succ = connection.event_connected.get() # Wait for connection - if not succ: raise Exception("Connection event return error") - return connection + except Exception, err: + self.log.debug("%s Connect error: %s" % (ip, Debug.formatException(err))) + connection.close() + raise err + return connection + else: + return None - # No connection found - if create: # Allow to create new connection if not found - if port == 0: - raise Exception("This peer is not connectable") - try: - connection = Connection(self, ip, port) - self.ips[ip] = connection - self.connections.append(connection) - succ = connection.connect() - if not succ: - connection.close() - raise Exception("Connection event return error") + def removeConnection(self, connection): + self.log.debug("Removing %s..." % connection) + if self.ips.get(connection.ip) == connection: # Delete if same as in registry + del self.ips[connection.ip] + if connection.peer_id and self.peer_ids.get(connection.peer_id) == connection: # Delete if same as in registry + del self.peer_ids[connection.peer_id] + if connection in self.connections: + self.connections.remove(connection) - except Exception, err: - self.log.debug("%s Connect error: %s" % (ip, Debug.formatException(err))) - connection.close() - raise err - return connection - else: - return None + def checkConnections(self): + while self.running: + time.sleep(60) # Sleep 1 min + self.ip_incoming = {} + for connection in self.connections[:]: # Make a copy + idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time) + if connection.unpacker and idle > 30: # Delete the unpacker if not needed + del connection.unpacker + connection.unpacker = None + connection.log("Unpacker deleted") + if idle > 60 * 60: # Wake up after 1h + connection.log("[Cleanup] After wakeup, idle: %s" % idle) + connection.close() - def removeConnection(self, connection): - self.log.debug("Removing %s..." % connection) - if self.ips.get(connection.ip) == connection: # Delete if same as in registry - del self.ips[connection.ip] - if connection.peer_id and self.peer_ids.get(connection.peer_id) == connection: # Delete if same as in registry - del self.peer_ids[connection.peer_id] - if connection in self.connections: - self.connections.remove(connection) + elif idle > 20 * 60 and connection.last_send_time < time.time() - 10: # Idle more than 20 min and we not send request in last 10 sec + if not connection.ping(): # send ping request + connection.close() + elif idle > 10 and connection.incomplete_buff_recv > 0: # Incompelte data with more than 10 sec idle + connection.log("[Cleanup] Connection buff stalled") + connection.close() + elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10: # Sent command and no response in 10 sec + connection.log("[Cleanup] Command %s timeout: %s" % (connection.last_cmd, time.time() - connection.last_send_time)) + connection.close() - def checkConnections(self): - while self.running: - time.sleep(60) # Sleep 1 min - for connection in self.connections[:]: # Make a copy - idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time) - - if connection.unpacker and idle > 30: # Delete the unpacker if not needed - del connection.unpacker - connection.unpacker = None - connection.log("Unpacker deleted") - - if idle > 60*60: # Wake up after 1h - connection.log("[Cleanup] After wakeup, idle: %s" % idle) - connection.close() - - elif idle > 20*60 and connection.last_send_time < time.time()-10: # Idle more than 20 min and we not send request in last 10 sec - if not connection.ping(): # send ping request - connection.close() - - elif idle > 10 and connection.incomplete_buff_recv > 0: # Incompelte data with more than 10 sec idle - connection.log("[Cleanup] Connection buff stalled") - connection.close() - - elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10: # Sent command and no response in 10 sec - connection.log("[Cleanup] Command %s timeout: %s" % (connection.last_cmd, time.time() - connection.last_send_time)) - connection.close() - - elif idle > 60 and connection.protocol == "?": # No connection after 1 min - connection.log("[Cleanup] Connect timeout: %s" % idle) - connection.close() - - -# -- TESTING -- - -def testCreateServer(): - global server - server = ConnectionServer("127.0.0.1", 1234, testRequestHandler) - server.start() - - -def testRequestHandler(connection, req): - print req - if req["cmd"] == "Bigdata": - connection.send({"res": "HelloWorld"*1024}) - else: - connection.send({"res": "pong"}) - - -def testClient(num): - time.sleep(1) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(("localhost", 1234)) - for i in range(10): - print "[C%s] send..." % num - s.sendall(msgpack.packb({"cmd": "[C] Ping"})) - print "[C%s] recv..." % num - print "[C%s] %s" % (num, repr(s.recv(1024))) - time.sleep(1) - - -def testSlowClient(num): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(("localhost", 1234)) - for i in range(1): - print "[C%s] send..." % num - s.sendall(msgpack.packb({"cmd": "Bigdata"})) - print "[C%s] recv..." % num - gevent.spawn_later(1, lambda s: s.send(msgpack.packb({"cmd": "[Z] Ping"})), s) - while 1: - data = s.recv(1000) - if not data: break - print "[C%s] %s" % (num, data) - time.sleep(1) - #s.sendall(msgpack.packb({"cmd": "[C] Ping"})) - - -def testZmqClient(num): - import zmq.green as zmq - c = zmq.Context(1) - for i in range(10): - s = c.socket(zmq.REQ) - s.connect('tcp://127.0.0.1:1234') - print "[Z%s] send..." % num - s.send(msgpack.packb({"cmd": "[Z] Ping %s" % i})) - print "[Z%s] recv..." % num - print "[Z%s] %s" % (num, s.recv(1024)) - s.close() - time.sleep(1) - - -def testZmqSlowClient(num): - import zmq.green as zmq - c = zmq.Context(1) - s = c.socket(zmq.REQ) - for i in range(1): - s.connect('tcp://127.0.0.1:1234') - print "[Z%s] send..." % num - s.send(msgpack.packb({"cmd": "Bigdata"})) - print "[Z%s] recv..." % num - #gevent.spawn_later(1, lambda s: s.send(msgpack.packb({"cmd": "[Z] Ping"})), s) - while 1: - data = s.recv(1024*1024) - if not data: break - print "[Z%s] %s" % (num, data) - time.sleep(1) - s.send(msgpack.packb({"cmd": "[Z] Ping"})) - - -def testConnection(): - global server - connection = server.getConnection("127.0.0.1", 1234) - connection.send({"res": "Sending: Hello!"}) - print connection - - -def greenletsNum(): - from greenlet import greenlet - import gc - while 1: - print len([ob for ob in gc.get_objects() if isinstance(ob, greenlet)]) - time.sleep(1) - -if __name__ == "__main__": - from gevent import monkey; monkey.patch_all(thread=False) - import sys, time - logging.getLogger().setLevel(logging.DEBUG) - - gevent.spawn(testZmqClient, 1) - gevent.spawn(greenletsNum) - #gevent.spawn(testClient, 1) - #gevent.spawn_later(1, testConnection) - print "Running server..." - server = None - testCreateServer() - + elif idle > 60 and connection.protocol == "?": # No connection after 1 min + connection.log("[Cleanup] Connect timeout: %s" % idle) + connection.close() diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index e84b9cd1..c461456f 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -1,20 +1,29 @@ -import json, time, re, os, gevent, copy +import json +import time +import re +import os +import copy + +import gevent + from Debug import Debug from Crypt import CryptHash from Config import config + class ContentManager(object): + def __init__(self, site): self.site = site self.log = self.site.log - self.contents = {} # Known content.json (without files and includes) + self.contents = {} # Known content.json (without files and includes) self.loadContent(add_bad_files=False) self.site.settings["size"] = self.getTotalSize() # Load content.json to self.content # Return: Changed files ["index.html", "data/messages.json"] def loadContent(self, content_inner_path="content.json", add_bad_files=True, load_includes=True): - content_inner_path = content_inner_path.strip("/") # Remove / from begning + content_inner_path = content_inner_path.strip("/") # Remove / from begning old_content = self.contents.get(content_inner_path) content_path = self.site.storage.getPath(content_inner_path) content_path_dir = self.toDir(self.site.storage.getPath(content_inner_path)) @@ -28,7 +37,7 @@ class ContentManager(object): return False else: self.log.error("Content.json not exist: %s" % content_path) - return False # Content.json not exist + return False # Content.json not exist try: # Get the files where the sha512 changed @@ -36,40 +45,44 @@ class ContentManager(object): for relative_path, info in new_content.get("files", {}).items(): if "sha512" in info: hash_type = "sha512" - else: # Backward compatiblity + else: # Backward compatiblity hash_type = "sha1" new_hash = info[hash_type] - if old_content and old_content["files"].get(relative_path): # We have the file in the old content + if old_content and old_content["files"].get(relative_path): # We have the file in the old content old_hash = old_content["files"][relative_path].get(hash_type) - else: # The file is not in the old content + else: # The file is not in the old content old_hash = None - if old_hash != new_hash: changed.append(content_dir+relative_path) + if old_hash != new_hash: + changed.append(content_dir + relative_path) # Load includes if load_includes and "includes" in new_content: for relative_path, info in new_content["includes"].items(): - include_inner_path = content_dir+relative_path - if self.site.storage.isFile(include_inner_path): # Content.json exists, load it + include_inner_path = content_dir + relative_path + if self.site.storage.isFile(include_inner_path): # Content.json exists, load it success = self.loadContent(include_inner_path, add_bad_files=add_bad_files) - if success: changed += success # Add changed files - else: # Content.json not exist, add to changed files + if success: + changed += success # Add changed files + else: # Content.json not exist, add to changed files self.log.debug("Missing include: %s" % include_inner_path) changed += [include_inner_path] # Load blind user includes (all subdir) if load_includes and "user_contents" in new_content: for relative_dir in os.listdir(content_path_dir): - include_inner_path = content_dir+relative_dir+"/content.json" - if not self.site.storage.isFile(include_inner_path): continue # Content.json not exist + include_inner_path = content_dir + relative_dir + "/content.json" + if not self.site.storage.isFile(include_inner_path): + continue # Content.json not exist success = self.loadContent(include_inner_path, add_bad_files=add_bad_files, load_includes=False) - if success: changed += success # Add changed files + if success: + changed += success # Add changed files # Update the content self.contents[content_inner_path] = new_content except Exception, err: self.log.error("Content.json parse error: %s" % Debug.formatException(err)) - return False # Content.json parse error + return False # Content.json parse error # Add changed files to bad files if add_bad_files: @@ -77,7 +90,8 @@ class ContentManager(object): self.site.bad_files[inner_path] = True if new_content["modified"] > self.site.settings.get("modified", 0): - self.site.settings["modified"] = min(time.time()+60*10, new_content["modified"]) # Dont store modifications in the far future (more than 10 minute) + # Dont store modifications in the far future (more than 10 minute) + self.site.settings["modified"] = min(time.time() + 60 * 10, new_content["modified"]) return changed @@ -86,8 +100,9 @@ class ContentManager(object): def getTotalSize(self, ignore=None): total_size = 0 for inner_path, content in self.contents.iteritems(): - if inner_path == ignore: continue - total_size += self.site.storage.getSize(inner_path) # Size of content.json + if inner_path == ignore: + continue + total_size += self.site.storage.getSize(inner_path) # Size of content.json for file, info in content.get("files", {}).iteritems(): total_size += info["size"] return total_size @@ -95,12 +110,12 @@ class ContentManager(object): # Find the file info line from self.contents # Return: { "sha512": "c29d73d30ee8c9c1b5600e8a84447a6de15a3c3db6869aca4a2a578c1721f518", "size": 41 , "content_inner_path": "content.json"} def getFileInfo(self, inner_path): - dirs = inner_path.split("/") # Parent dirs of content.json - inner_path_parts = [dirs.pop()] # Filename relative to content.json + dirs = inner_path.split("/") # Parent dirs of content.json + inner_path_parts = [dirs.pop()] # Filename relative to content.json while True: content_inner_path = "%s/content.json" % "/".join(dirs) content = self.contents.get(content_inner_path.strip("/")) - if content and "files" in content: # Check if content.json exists + if content and "files" in content: # Check if content.json exists back = content["files"].get("/".join(inner_path_parts)) if back: back["content_inner_path"] = content_inner_path @@ -124,13 +139,14 @@ class ContentManager(object): # Get rules for the file # Return: The rules for the file or False if not allowed def getRules(self, inner_path, content=None): - if not inner_path.endswith("content.json"): # Find the files content.json first + if not inner_path.endswith("content.json"): # Find the files content.json first file_info = self.getFileInfo(inner_path) - if not file_info: return False # File not found + if not file_info: + return False # File not found inner_path = file_info["content_inner_path"] - dirs = inner_path.split("/") # Parent dirs of content.json - inner_path_parts = [dirs.pop()] # Filename relative to content.json - inner_path_parts.insert(0, dirs.pop()) # Dont check in self dir + dirs = inner_path.split("/") # Parent dirs of content.json + inner_path_parts = [dirs.pop()] # Filename relative to content.json + inner_path_parts.insert(0, dirs.pop()) # Dont check in self dir while True: content_inner_path = "%s/content.json" % "/".join(dirs) parent_content = self.contents.get(content_inner_path.strip("/")) @@ -138,23 +154,23 @@ class ContentManager(object): return parent_content["includes"].get("/".join(inner_path_parts)) elif parent_content and "user_contents" in parent_content: return self.getUserContentRules(parent_content, inner_path, content) - else: # No inner path in this dir, lets try the parent dir + else: # No inner path in this dir, lets try the parent dir if dirs: inner_path_parts.insert(0, dirs.pop()) - else: # No more parent dirs + else: # No more parent dirs break return False - # Get rules for a user file # Return: The rules of the file or False if not allowed def getUserContentRules(self, parent_content, inner_path, content): user_contents = parent_content["user_contents"] - user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) # Delivered for directory + user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1) # Delivered for directory try: - if not content: content = self.site.storage.loadJson(inner_path) # Read the file if no content specified + if not content: + content = self.site.storage.loadJson(inner_path) # Read the file if no content specified except (Exception, ): # Content.json not exist return {"signers": [user_address], "user_address": user_address} # Return information that we know for sure @@ -162,15 +178,16 @@ class ContentManager(object): content["cert_auth_type"] = "unknown" content["cert_user_name"] = "unknown@unknown" """ - user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit + user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"]) # web/nofish@zeroid.bit - rules = copy.copy(user_contents["permissions"].get(content["cert_user_id"], {})) # Default rules by username - if rules == False: + rules = copy.copy(user_contents["permissions"].get(content["cert_user_id"], {})) # Default rules by username + if rules is False: return False # User banned if "signers" in rules: rules["signers"] = rules["signers"][:] # Make copy of the signers - for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules - if not re.match(permission_pattern, user_urn): continue # Rule is not valid for user + for permission_pattern, permission_rules in user_contents["permission_rules"].items(): # Regexp rules + if not re.match(permission_pattern, user_urn): + continue # Rule is not valid for user # Update rules if its better than current recorded ones for key, val in permission_rules.iteritems(): if key not in rules: @@ -182,13 +199,15 @@ class ContentManager(object): if val > rules[key]: rules[key] = val elif hasattr(val, "startswith"): # String, update if longer - if len(val) > len(rules[key]): rules[key] = val + if len(val) > len(rules[key]): + rules[key] = val elif type(val) is list: # List, append rules[key] += val - rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers - if "signers" not in rules: rules["signers"] = [] - rules["signers"].append(user_address) # Add user as valid signer + rules["cert_signers"] = user_contents["cert_signers"] # Add valid cert signers + if "signers" not in rules: + rules["signers"] = [] + rules["signers"].append(user_address) # Add user as valid signer rules["user_address"] = user_address return rules @@ -197,7 +216,7 @@ class ContentManager(object): # Return: The new content if filewrite = False def sign(self, inner_path="content.json", privatekey=None, filewrite=True, update_changed_files=False, extend=None): content = self.contents.get(inner_path) - if not content: # Content not exist yet, load default one + if not content: # Content not exist yet, load default one self.log.info("File %s not exist yet, loading default values..." % inner_path) content = {"files": {}, "signs": {}} # Default content.json if inner_path == "content.json": # It's the root content.json, add some more fields @@ -205,7 +224,8 @@ class ContentManager(object): content["description"] = "" content["signs_required"] = 1 content["ignore"] = "" - if extend: content.update(extend) # Add custom fields + if extend: + content.update(extend) # Add custom fields directory = self.toDir(self.site.storage.getPath(inner_path)) self.log.info("Opening site data directory: %s..." % directory) @@ -217,18 +237,25 @@ class ContentManager(object): file_path = self.site.storage.getPath("%s/%s" % (root.strip("/"), file_name)) file_inner_path = re.sub(re.escape(directory), "", file_path) - if file_name == "content.json": ignored = True - elif content.get("ignore") and re.match(content["ignore"], file_inner_path): ignored = True - elif file_name.startswith("."): ignored = True - else: ignored = False + if file_name == "content.json": + ignored = True + elif content.get("ignore") and re.match(content["ignore"], file_inner_path): + ignored = True + elif file_name.startswith("."): + ignored = True + else: + ignored = False - if ignored: # Ignore content.json, definied regexp and files starting with . + if ignored: # Ignore content.json, definied regexp and files starting with . self.log.info("- [SKIPPED] %s" % file_inner_path) else: - sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file + sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file self.log.info("- %s (SHA512: %s)" % (file_inner_path, sha512sum)) hashed_files[file_inner_path] = {"sha512": sha512sum, "size": os.path.getsize(file_path)} - if file_inner_path in content["files"].keys() and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512"): + if ( + file_inner_path in content["files"].keys() + and hashed_files[file_inner_path]["sha512"] != content["files"][file_inner_path].get("sha512") + ): changed_files.append(file_path) self.log.debug("Changed files: %s" % changed_files) @@ -239,9 +266,9 @@ class ContentManager(object): # Generate new content.json self.log.info("Adding timestamp and sha512sums to new content.json...") - new_content = content.copy() # Create a copy of current content.json - new_content["files"] = hashed_files # Add files sha512 hash - new_content["modified"] = time.time() # Add timestamp + new_content = content.copy() # Create a copy of current content.json + new_content["files"] = hashed_files # Add files sha512 hash + new_content["modified"] = time.time() # Add timestamp if inner_path == "content.json": new_content["address"] = self.site.address new_content["zeronet_version"] = config.version @@ -255,19 +282,22 @@ class ContentManager(object): return self.log.error("Private key invalid! Valid signers: %s, Private key address: %s" % (valid_signers, privatekey_address)) self.log.info("Correct %s in valid signers: %s" % (privatekey_address, valid_signers)) - if inner_path == "content.json" and privatekey_address == self.site.address: # If signing using the root key sign the valid signers + if inner_path == "content.json" and privatekey_address == self.site.address: # If signing using the root key sign the valid signers new_content["signers_sign"] = CryptBitcoin.sign("%s:%s" % (new_content["signs_required"], ",".join(valid_signers)), privatekey) - if not new_content["signers_sign"]: self.log.info("Old style address, signers_sign is none") + if not new_content["signers_sign"]: + self.log.info("Old style address, signers_sign is none") self.log.info("Signing %s..." % inner_path) - if "signs" in new_content: del(new_content["signs"]) # Delete old signs - if "sign" in new_content: del(new_content["sign"]) # Delete old sign (backward compatibility) + if "signs" in new_content: + del(new_content["signs"]) # Delete old signs + if "sign" in new_content: + del(new_content["sign"]) # Delete old sign (backward compatibility) sign_content = json.dumps(new_content, sort_keys=True) sign = CryptBitcoin.sign(sign_content, privatekey) - #new_content["signs"] = content.get("signs", {}) # TODO: Multisig - if sign: # If signing is successful (not an old address) + # new_content["signs"] = content.get("signs", {}) # TODO: Multisig + if sign: # If signing is successful (not an old address) new_content["signs"] = {} new_content["signs"][privatekey_address] = sign @@ -294,7 +324,7 @@ class ContentManager(object): # Return: ["1KRxE1s3oDyNDawuYWpzbLUwNm8oDbeEp6", "13ReyhCsjhpuCVahn1DHdf6eMqqEVev162"] def getValidSigners(self, inner_path, content=None): valid_signers = [] - if inner_path == "content.json": # Root content.json + if inner_path == "content.json": # Root content.json if "content.json" in self.contents and "signers" in self.contents["content.json"]: valid_signers += self.contents["content.json"]["signers"].keys() else: @@ -314,11 +344,12 @@ class ContentManager(object): from Crypt import CryptBitcoin rules = self.getRules(inner_path, content) - if not rules.get("cert_signers"): return True # Does not need cert + if not rules.get("cert_signers"): + return True # Does not need cert name, domain = content["cert_user_id"].split("@") cert_address = rules["cert_signers"].get(domain) - if not cert_address: # Cert signer not allowed + if not cert_address: # Cert signer not allowed self.log.error("Invalid cert signer: %s" % domain) return False return CryptBitcoin.verify("%s#%s/%s" % (rules["user_address"], content["cert_auth_type"], name), cert_address, content["cert_sign"]) @@ -326,21 +357,23 @@ class ContentManager(object): # Checks if the content.json content is valid # Return: True or False def validContent(self, inner_path, content): - content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content - site_size = self.getTotalSize(ignore=inner_path)+content_size # Site size without old content - if site_size > self.site.settings.get("size", 0): self.site.settings["size"] = site_size # Save to settings if larger + content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content + site_size = self.getTotalSize(ignore=inner_path) + content_size # Site size without old content + if site_size > self.site.settings.get("size", 0): + self.site.settings["size"] = site_size # Save to settings if larger - site_size_limit = self.site.getSizeLimit()*1024*1024 + site_size_limit = self.site.getSizeLimit() * 1024 * 1024 # Check total site size limit if site_size > site_size_limit: self.log.error("%s: Site too large %s > %s, aborting task..." % (inner_path, site_size, site_size_limit)) task = self.site.worker_manager.findTask(inner_path) - if task: # Dont try to download from other peers + if task: # Dont try to download from other peers self.site.worker_manager.failTask(task) return False - if inner_path == "content.json": return True # Root content.json is passed + if inner_path == "content.json": + return True # Root content.json is passed # Load include details rules = self.getRules(inner_path, content) @@ -349,15 +382,15 @@ class ContentManager(object): return False # Check include size limit - if rules.get("max_size"): # Include size limit + if rules.get("max_size"): # Include size limit if content_size > rules["max_size"]: self.log.error("%s: Include too large %s > %s" % (inner_path, content_size, rules["max_size"])) return False # Check if content includes allowed - if rules.get("includes_allowed") == False and content.get("includes"): + if rules.get("includes_allowed") is False and content.get("includes"): self.log.error("%s: Includes not allowed" % inner_path) - return False # Includes not allowed + return False # Includes not allowed # Filename limit if rules.get("files_allowed"): @@ -370,54 +403,57 @@ class ContentManager(object): # Verify file validity # Return: None = Same as before, False = Invalid, True = Valid - def verifyFile(self, inner_path, file, ignore_same = True): - if inner_path.endswith("content.json"): # content.json: Check using sign + def verifyFile(self, inner_path, file, ignore_same=True): + if inner_path.endswith("content.json"): # content.json: Check using sign from Crypt import CryptBitcoin try: new_content = json.load(file) if inner_path in self.contents: old_content = self.contents.get(inner_path) # Checks if its newer the ours - if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json + if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json return None - elif old_content["modified"] > new_content["modified"]: # We have newer + elif old_content["modified"] > new_content["modified"]: # We have newer self.log.debug("We have newer %s (Our: %s, Sent: %s)" % (inner_path, old_content["modified"], new_content["modified"])) - gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers + gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers return False - if new_content["modified"] > time.time()+60*60*24: # Content modified in the far future (allow 1 day window) + if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day window) self.log.error("%s modify is in the future!" % inner_path) return False # Check sign sign = new_content.get("sign") signs = new_content.get("signs", {}) - if "sign" in new_content: del(new_content["sign"]) # The file signed without the sign - if "signs" in new_content: del(new_content["signs"]) # The file signed without the signs - sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace + if "sign" in new_content: + del(new_content["sign"]) # The file signed without the sign + if "signs" in new_content: + del(new_content["signs"]) # The file signed without the signs + sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace - if not self.validContent(inner_path, new_content): return False # Content not valid (files too large, invalid files) + if not self.validContent(inner_path, new_content): + return False # Content not valid (files too large, invalid files) - if signs: # New style signing + if signs: # New style signing valid_signers = self.getValidSigners(inner_path, new_content) signs_required = self.getSignsRequired(inner_path, new_content) - if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json + if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json if not CryptBitcoin.verify("%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]): self.log.error("%s invalid signers_sign!" % inner_path) return False - if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid + if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid self.log.error("%s invalid cert!" % inner_path) return False valid_signs = 0 for address in valid_signers: - if address in signs: valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) - if valid_signs >= signs_required: break # Break if we has enough signs - - + if address in signs: + valid_signs += CryptBitcoin.verify(sign_content, address, signs[address]) + if valid_signs >= signs_required: + break # Break if we has enough signs return valid_signs >= signs_required - else: # Old style signing + else: # Old style signing return CryptBitcoin.verify(sign_content, self.site.address, sign) except Exception, err: @@ -429,7 +465,7 @@ class ContentManager(object): if file_info: if "sha512" in file_info: hash_valid = CryptHash.sha512sum(file) == file_info["sha512"] - elif "sha1" in file_info: # Backward compatibility + elif "sha1" in file_info: # Backward compatibility hash_valid = CryptHash.sha1sum(file) == file_info["sha1"] else: hash_valid = False @@ -439,22 +475,21 @@ class ContentManager(object): return False return hash_valid - else: # File not in content.json + else: # File not in content.json self.log.error("File not in content.json: %s" % inner_path) return False - # Get dir from file # Return: data/site/content.json -> data/site def toDir(self, inner_path): file_dir = re.sub("[^/]*?$", "", inner_path).strip("/") - if file_dir: file_dir += "/" # Add / at end if its not the root + if file_dir: + file_dir += "/" # Add / at end if its not the root return file_dir def testSign(): global config - from Config import config from Site import Site site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH") content_manager = ContentManager(site) @@ -462,9 +497,7 @@ def testSign(): def testVerify(): - from Config import config from Site import Site - #site = Site("1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr") site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH") content_manager = ContentManager(site) @@ -478,7 +511,6 @@ def testVerify(): def testInfo(): - from Config import config from Site import Site site = Site("12Hw8rTgzrNo4DSh2AkqwPRqDyTticwJyH") @@ -493,14 +525,13 @@ def testInfo(): if __name__ == "__main__": - import os, sys, logging + import sys + import logging os.chdir("../..") sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("src")) logging.basicConfig(level=logging.DEBUG) - from Debug import Debug - from Crypt import CryptHash - #testSign() + # testSign() testVerify() - #testInfo() + # testInfo() diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index 5c57d2df..74a2d97d 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -185,7 +185,7 @@ class Peer(object): # Stop and remove from site def remove(self): self.log("Removing peer...Connection error: %s, Hash failed: %s" % (self.connection_error, self.hash_failed)) - if self.key in self.site.peers: del(self.site.peers[self.key]) + if self.site and self.key in self.site.peers: del(self.site.peers[self.key]) if self.connection: self.connection.close() diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee index 8afa2cb1..deb7a823 100644 --- a/src/Ui/media/Wrapper.coffee +++ b/src/Ui/media/Wrapper.coffee @@ -242,7 +242,12 @@ class Wrapper @setSiteInfo site_info if site_info.settings.size > site_info.size_limit*1024*1024 # Site size too large and not displaying it yet - @loading.showTooLarge(site_info) + if @loading.screen_visible + @loading.showTooLarge(site_info) + else + @displayConfirm "Site is larger than allowed: #{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB", "Set limit to #{site_info.next_size_limit}MB", => + @ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) => + @notifications.add("size_limit", "done", res, 5000) if site_info.content window.document.title = site_info.content.title+" - ZeroNet" @@ -286,8 +291,8 @@ class Wrapper @loading.printLine "No peers found" if not @site_info and not @loading.screen_visible and $("#inner-iframe").attr("src").indexOf("?") == -1 # First site info and mainpage - if site_info.size_limit < site_info.next_size_limit # Need upgrade soon - @wrapperConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", => + if site_info.size_limit*1.1 < site_info.next_size_limit # Need upgrade soon + @actionConfirm "Running out of size limit (#{(site_info.settings.size/1024/1024).toFixed(1)}MB/#{site_info.size_limit}MB)", "Set limit to #{site_info.next_size_limit}MB", => @ws.cmd "siteSetLimit", [site_info.next_size_limit], (res) => @notifications.add("size_limit", "done", res, 5000) return false diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js index 5b5559ac..0c5693fe 100644 --- a/src/Ui/media/all.js +++ b/src/Ui/media/all.js @@ -681,7 +681,6 @@ jQuery.extend( jQuery.easing, }).call(this); - /* ---- src/Ui/media/Sidebar.coffee ---- */ @@ -1069,7 +1068,15 @@ jQuery.extend( jQuery.easing, _this.address = site_info.address; _this.setSiteInfo(site_info); if (site_info.settings.size > site_info.size_limit * 1024 * 1024) { - _this.loading.showTooLarge(site_info); + if (_this.loading.screen_visible) { + _this.loading.showTooLarge(site_info); + } else { + _this.displayConfirm("Site is larger than allowed: " + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB", "Set limit to " + site_info.next_size_limit + "MB", function() { + return _this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) { + return _this.notifications.add("size_limit", "done", res, 5000); + }); + }); + } } if (site_info.content) { window.document.title = site_info.content.title + " - ZeroNet"; @@ -1118,8 +1125,8 @@ jQuery.extend( jQuery.easing, } } if (!this.site_info && !this.loading.screen_visible && $("#inner-iframe").attr("src").indexOf("?") === -1) { - if (site_info.size_limit < site_info.next_size_limit) { - this.wrapperConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) { + if (site_info.size_limit * 1.1 < site_info.next_size_limit) { + this.actionConfirm("Running out of size limit (" + ((site_info.settings.size / 1024 / 1024).toFixed(1)) + "MB/" + site_info.size_limit + "MB)", "Set limit to " + site_info.next_size_limit + "MB", (function(_this) { return function() { _this.ws.cmd("siteSetLimit", [site_info.next_size_limit], function(res) { return _this.notifications.add("size_limit", "done", res, 5000); @@ -1211,4 +1218,4 @@ jQuery.extend( jQuery.easing, window.wrapper = new Wrapper(ws_url); -}).call(this); \ No newline at end of file +}).call(this);