diff --git a/README.md b/README.md index 2eba782e..b58fc90a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Decentralized websites using Bitcoin crypto and BitTorrent network - After starting `zeronet.py` you will be able to visit zeronet sites using http://127.0.0.1:43110/{zeronet_address} (eg. http://127.0.0.1:43110/1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr). - When you visit a new zeronet site, it's trying to find peers using BitTorrent network and download the site files (html, css, js...) from them. - Each visited sites become also served by You. - - Every site containing a `site.json` which holds all other files sha1 hash and a sign generated using site's private key. + - Every site containing a `site.json` which holds all other files sha512 hash and a sign generated using site's private key. - If the site owner (who has the private key for the site address) modifies the site, then he/she signs the new `content.json` and publish it to the peers. After the peers have verified the `content.json` integrity (using the sign), they download the modified files and publish the new content to other peers. diff --git a/src/Config.py b/src/Config.py index a5ca95c9..99ad9567 100644 --- a/src/Config.py +++ b/src/Config.py @@ -3,7 +3,7 @@ import ConfigParser class Config(object): def __init__(self): - self.version = "0.2.0" + self.version = "0.2.1" self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file @@ -60,7 +60,9 @@ class Config(object): parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port') parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip') + parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name') parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address') + parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit') parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port') diff --git a/src/Content/ContentManager.py b/src/Content/ContentManager.py index 96e49671..2e69252d 100644 --- a/src/Content/ContentManager.py +++ b/src/Content/ContentManager.py @@ -9,6 +9,7 @@ class ContentManager: self.log = self.site.log self.contents = {} # Known content.json (without files and includes) self.loadContent(add_bad_files = False) + self.site.settings["size"] = self.getTotalSize() # Load content.json to self.content @@ -68,9 +69,24 @@ class ContentManager: for inner_path in changed: self.site.bad_files[inner_path] = True + if new_content["modified"] > self.site.settings.get("modified", 0): + self.site.settings["modified"] = new_content["modified"] + return changed + # Get total size of site + # Return: 32819 (size of files in kb) + def getTotalSize(self, ignore=None): + total_size = 0 + for inner_path, content in self.contents.iteritems(): + if inner_path == ignore: continue + total_size += os.path.getsize(self.site.getPath(inner_path)) # Size of content.json + for file, info in content.get("files", {}).iteritems(): + total_size += info["size"] + return total_size + + # Find the file info line from self.contents # Return: { "sha512": "c29d73d30ee8c9c1b5600e8a84447a6de15a3c3db6869aca4a2a578c1721f518", "size": 41 , "content_inner_path": "content.json"} def getFileInfo(self, inner_path): @@ -216,30 +232,50 @@ class ContentManager: return 1 # Todo: Multisig + # Checks if the content.json content is valid + # Return: True or False def validContent(self, inner_path, content): - if inner_path == "content.json": return True # Always ok + content_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) # Size of new content + site_size = self.getTotalSize(ignore=inner_path)+content_size # Site size without old content + if site_size > self.site.settings.get("size", 0): self.site.settings["size"] = site_size # Save to settings if larger + + site_size_limit = self.site.getSizeLimit()*1024*1024 + + # Check total site size limit + if site_size > site_size_limit: + self.log.error("%s: Site too large %s > %s, aborting task..." % (inner_path, site_size, site_size_limit)) + task = self.site.worker_manager.findTask(inner_path) + if task: # Dont try to download from other peers + self.site.worker_manager.failTask(task) + return False + + if inner_path == "content.json": return True # Root content.json is passed + + # Load include details include_info = self.getIncludeInfo(inner_path) if not include_info: self.log.error("%s: No include info" % inner_path) return False - if include_info.get("max_size"): # Size limit - total_size = len(json.dumps(content)) + sum([file["size"] for file in content["files"].values()]) - if total_size > include_info["max_size"]: - self.log.error("%s: Too large %s > %s" % (inner_path, total_size, include_info["max_size"])) + # Check include size limit + if include_info.get("max_size"): # Include size limit + if content_size > include_info["max_size"]: + self.log.error("%s: Include too large %s > %s" % (inner_path, total_size, include_info["max_size"])) return False + # Check if content includes allowed if include_info.get("includes_allowed") == False and content.get("includes"): self.log.error("%s: Includes not allowed" % inner_path) return False # Includes not allowed - if include_info.get("files_allowed"): # Filename limit + # Filename limit + if include_info.get("files_allowed"): for file_inner_path in content["files"].keys(): if not re.match("^%s$" % include_info["files_allowed"], file_inner_path): self.log.error("%s: File not allowed: " % file_inner_path) return False - return True + return True # All good @@ -292,7 +328,7 @@ class ContentManager: self.log.error("Verify sign error: %s" % Debug.formatException(err)) return False - else: # Check using sha1 hash + else: # Check using sha512 hash file_info = self.getFileInfo(inner_path) if file_info: if "sha512" in file_info: diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index 050ca240..02b134e1 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -41,7 +41,9 @@ class FileRequest: return False if site.settings["own"] and params["inner_path"].endswith("content.json"): self.log.debug("Someone trying to push a file to own site %s, reload local %s first" % (site.address, params["inner_path"])) - site.content_manager.loadContent(params["inner_path"]) + changed = site.content_manager.loadContent(params["inner_path"], add_bad_files=False) + if changed: # Content.json changed locally + site.settings["size"] = site.content_manager.getTotalSize() # Update site size buff = StringIO(params["body"]) valid = site.content_manager.verifyFile(params["inner_path"], buff) if valid == True: # Valid and changed diff --git a/src/Site/Site.py b/src/Site/Site.py index 56b5dce3..21cf7105 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -33,8 +33,8 @@ class Site: self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] self.page_requested = False # Page viewed in browser - self.content_manager = ContentManager(self) # Load contents self.loadSettings() # Load settings from sites.json + self.content_manager = ContentManager(self) # Load contents if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed) self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24)) @@ -74,6 +74,22 @@ class Site: return + # Max site size in MB + def getSizeLimit(self): + return self.settings.get("size_limit", config.size_limit) + + + # Next size limit based on current size + def getNextSizeLimit(self): + size_limits = [10,20,50,100,200,500,1000,2000,5000,10000,20000,50000,100000] + size = self.settings.get("size", 0) + for size_limit in size_limits: + if size*1.2 < size_limit*1024*1024: + return size_limit + return 999999 + + + # Sercurity check and return path of site's file def getPath(self, inner_path): inner_path = inner_path.replace("\\", "/") # Windows separator fix @@ -123,10 +139,14 @@ class Site: # Download all files of the site @util.Noparallel(blocking=False) - def download(self): + def download(self, check_size=False): self.log.debug("Start downloading...%s" % self.bad_files) self.announce() self.last_downloads = [] + if check_size: # Check the size first + valid = downloadContent(download_files=False) + if not valid: return False # Cant download content.jsons or size is not fits + found = self.downloadContent("content.json") return found @@ -147,6 +167,8 @@ class Site: if not self.settings["own"]: self.checkFiles(quick_check=True) # Quick check files based on file size if self.bad_files: self.download() + + self.settings["size"] = self.content_manager.getTotalSize() # Update site size return changed @@ -266,6 +288,8 @@ class Site: if added: self.worker_manager.onPeers() self.updateWebsocket(peers_added=added) + self.settings["peers"] = len(peers) + self.saveSettings() self.log.debug("Found %s peers, new: %s" % (len(peers), added)) else: pass # TODO: http tracker support diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 4692cfce..87d1b0d3 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -43,6 +43,7 @@ def isAddress(address): # Return site and start download site files def need(address, all_file=True): from Site import Site + new = False if address not in sites: # Site not exits yet if not isAddress(address): return False # Not address: %s % address logging.debug("Added new site: %s" % address) @@ -50,6 +51,7 @@ def need(address, all_file=True): if not sites[address].settings["serving"]: # Maybe it was deleted before sites[address].settings["serving"] = True sites[address].saveSettings() + new = True site = sites[address] if all_file: site.download() diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index e14a72bc..cc15d8e2 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -21,7 +21,10 @@ class UiWSGIHandler(WSGIHandler): self.ws_handler.run_application() else: # Standard HTTP request #print self.application.__class__.__name__ - return super(UiWSGIHandler, self).run_application() + try: + return super(UiWSGIHandler, self).run_application() + except Exception, err: + logging.debug("UiWSGIHandler error: %s" % err) class UiServer: @@ -77,5 +80,13 @@ class UiServer: self.log.info("Web interface: http://%s:%s/" % (config.ui_ip, config.ui_port)) self.log.info("--------------------------------------") + if config.open_browser: + logging.info("Opening browser: %s...", config.open_browser) + import webbrowser + if config.open_browser == "default_browser": + browser = webbrowser.get() + else: + browser = webbrowser.get(config.open_browser) + browser.open("http://%s:%s" % (config.ui_ip, config.ui_port), new=2) WSGIServer((self.ip, self.port), handler, handler_class=UiWSGIHandler, log=self.log).serve_forever() diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index c63a8978..6c556eab 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -84,6 +84,9 @@ class UiWebsocket: cmd = req.get("cmd") params = req.get("params") permissions = self.site.settings["permissions"] + if req["id"] >= 1000000: # Its a wrapper command, allow admin commands + permissions = permissions[:] + permissions.append("ADMIN") if cmd == "response": # It's a response to a command return self.actionResponse(req["to"], req["result"]) @@ -114,6 +117,8 @@ class UiWebsocket: func = self.actionSiteDelete elif cmd == "siteList" and "ADMIN" in permissions: func = self.actionSiteList + elif cmd == "siteSetLimit" and "ADMIN" in permissions: + func = self.actionSiteSetLimit elif cmd == "channelJoinAllsite" and "ADMIN" in permissions: func = self.actionChannelJoinAllsite # Unknown command @@ -155,16 +160,21 @@ class UiWebsocket: if "sign" in content: del(content["sign"]) if "signs" in content: del(content["signs"]) + settings = site.settings.copy() + del settings["wrapper_key"] # Dont expose wrapper key + ret = { "auth_key": self.site.settings["auth_key"], # Obsolete, will be removed "auth_key_sha512": hashlib.sha512(self.site.settings["auth_key"]).hexdigest()[0:64], # Obsolete, will be removed "auth_address": self.user.getAuthAddress(site.address), "address": site.address, - "settings": site.settings, + "settings": settings, "content_updated": site.content_updated, "bad_files": len(site.bad_files), + "size_limit": site.getSizeLimit(), + "next_size_limit": site.getNextSizeLimit(), "last_downloads": len(site.last_downloads), - "peers": len(site.peers), + "peers": site.settings["peers"], "tasks": len([task["inner_path"] for task in site.worker_manager.tasks]), "content": content } @@ -344,3 +354,10 @@ class UiWebsocket: site.updateWebsocket() else: self.response(to, {"error": "Unknown site: %s" % address}) + + + def actionSiteSetLimit(self, to, size_limit): + self.site.settings["size_limit"] = size_limit + self.site.saveSettings() + self.response(to, "Site size limit changed to %sMB" % size_limit) + self.site.download() diff --git a/src/Ui/media/Loading.coffee b/src/Ui/media/Loading.coffee index c42ed1df..2da7a39f 100644 --- a/src/Ui/media/Loading.coffee +++ b/src/Ui/media/Loading.coffee @@ -34,7 +34,10 @@ class Loading if not @screen_visible then return false $(".loadingscreen .console .cursor").remove() # Remove previous cursor if type == "error" then text = "#{text}" else text = text+" " - $(".loadingscreen .console").append("