import time import os import subprocess import shutil import collections import math import gevent import gevent.lock from Plugin import PluginManager from Debug import Debug from Crypt import CryptHash from lib import merkletools from util import helper import util from .BigfilePiecefield import BigfilePiecefield, BigfilePiecefieldPacked # We can only import plugin host clases after the plugins are loaded @PluginManager.afterLoad def importPluginnedClasses(): global VerifyError, config from Content.ContentManager import VerifyError from Config import config if "upload_nonces" not in locals(): upload_nonces = {} @PluginManager.registerTo("UiRequest") class UiRequestPlugin(object): def isCorsAllowed(self, path): if path == "/ZeroNet-Internal/BigfileUpload": return True else: return super(UiRequestPlugin, self).isCorsAllowed(path) def actionBigfileUpload(self): nonce = self.get.get("upload_nonce") if nonce not in upload_nonces: return self.error403("Upload nonce error.") upload_info = upload_nonces[nonce] del upload_nonces[nonce] self.sendHeader(200, "text/html", noscript=True, extra_headers={ "Access-Control-Allow-Origin": "null", "Access-Control-Allow-Credentials": "true" }) self.readMultipartHeaders(self.env['wsgi.input']) # Skip http headers site = upload_info["site"] inner_path = upload_info["inner_path"] with site.storage.open(inner_path, "wb", create_dirs=True) as out_file: merkle_root, piece_size, piecemap_info = site.content_manager.hashBigfile( self.env['wsgi.input'], upload_info["size"], upload_info["piece_size"], out_file ) if len(piecemap_info["sha512_pieces"]) == 1: # Small file, don't split hash = piecemap_info["sha512_pieces"][0].hex() hash_id = site.content_manager.hashfield.getHashId(hash) site.content_manager.optionalDownloaded(inner_path, hash_id, upload_info["size"], own=True) else: # Big file file_name = helper.getFilename(inner_path) msgpack.pack({file_name: piecemap_info}, site.storage.open(upload_info["piecemap"], "wb")) # Find piecemap and file relative path to content.json file_info = site.content_manager.getFileInfo(inner_path, new_file=True) content_inner_path_dir = helper.getDirname(file_info["content_inner_path"]) piecemap_relative_path = upload_info["piecemap"][len(content_inner_path_dir):] file_relative_path = inner_path[len(content_inner_path_dir):] # Add file to content.json if site.storage.isFile(file_info["content_inner_path"]): content = site.storage.loadJson(file_info["content_inner_path"]) else: content = {} if "files_optional" not in content: content["files_optional"] = {} content["files_optional"][file_relative_path] = { "sha512": merkle_root, "size": upload_info["size"], "piecemap": piecemap_relative_path, "piece_size": piece_size } merkle_root_hash_id = site.content_manager.hashfield.getHashId(merkle_root) site.content_manager.optionalDownloaded(inner_path, merkle_root_hash_id, upload_info["size"], own=True) site.storage.writeJson(file_info["content_inner_path"], content) site.content_manager.contents.loadItem(file_info["content_inner_path"]) # reload cache return { "merkle_root": merkle_root, "piece_num": len(piecemap_info["sha512_pieces"]), "piece_size": piece_size, "inner_path": inner_path } def readMultipartHeaders(self, wsgi_input): for i in range(100): line = wsgi_input.readline() if line == "\r\n": break return i def actionFile(self, file_path, *args, **kwargs): if kwargs.get("file_size", 0) > 1024 * 1024 and kwargs.get("path_parts"): # Only check files larger than 1MB path_parts = kwargs["path_parts"] site = self.server.site_manager.get(path_parts["address"]) big_file = site.storage.openBigfile(path_parts["inner_path"], prebuffer=2 * 1024 * 1024) if big_file: kwargs["file_obj"] = big_file kwargs["file_size"] = big_file.size return super(UiRequestPlugin, self).actionFile(file_path, *args, **kwargs) @PluginManager.registerTo("UiWebsocket") class UiWebsocketPlugin(object): def actionBigfileUploadInit(self, to, inner_path, size): valid_signers = self.site.content_manager.getValidSigners(inner_path) auth_address = self.user.getAuthAddress(self.site.address) if not self.site.settings["own"] and auth_address not in valid_signers: self.log.error("FileWrite forbidden %s not in valid_signers %s" % (auth_address, valid_signers)) return self.response(to, {"error": "Forbidden, you can only modify your own files"}) nonce = CryptHash.random() piece_size = 1024 * 1024 inner_path = self.site.content_manager.sanitizePath(inner_path) file_info = self.site.content_manager.getFileInfo(inner_path, new_file=True) content_inner_path_dir = helper.getDirname(file_info["content_inner_path"]) file_relative_path = inner_path[len(content_inner_path_dir):] upload_nonces[nonce] = { "added": time.time(), "site": self.site, "inner_path": inner_path, "websocket_client": self, "size": size, "piece_size": piece_size, "piecemap": inner_path + ".piecemap.msgpack" } return { "url": "/ZeroNet-Internal/BigfileUpload?upload_nonce=" + nonce, "piece_size": piece_size, "inner_path": inner_path, "file_relative_path": file_relative_path } def actionSiteSetAutodownloadBigfileLimit(self, to, limit): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") self.site.settings["autodownload_bigfile_size_limit"] = int(limit) self.response(to, "ok") def actionFileDelete(self, to, inner_path): piecemap_inner_path = inner_path + ".piecemap.msgpack" if self.hasFilePermission(inner_path) and self.site.storage.isFile(piecemap_inner_path): # Also delete .piecemap.msgpack file if exists self.log.debug("Deleting piecemap: %s" % piecemap_inner_path) file_info = self.site.content_manager.getFileInfo(piecemap_inner_path) if file_info: content_json = self.site.storage.loadJson(file_info["content_inner_path"]) relative_path = file_info["relative_path"] if relative_path in content_json.get("files_optional", {}): del content_json["files_optional"][relative_path] self.site.storage.writeJson(file_info["content_inner_path"], content_json) self.site.content_manager.loadContent(file_info["content_inner_path"], add_bad_files=False, force=True) try: self.site.storage.delete(piecemap_inner_path) except Exception as err: self.log.error("File %s delete error: %s" % (piecemap_inner_path, err)) return super(UiWebsocketPlugin, self).actionFileDelete(to, inner_path) @PluginManager.registerTo("ContentManager") class ContentManagerPlugin(object): def getFileInfo(self, inner_path, *args, **kwargs): if "|" not in inner_path: return super(ContentManagerPlugin, self).getFileInfo(inner_path, *args, **kwargs) inner_path, file_range = inner_path.split("|") pos_from, pos_to = map(int, file_range.split("-")) file_info = super(ContentManagerPlugin, self).getFileInfo(inner_path, *args, **kwargs) return file_info def readFile(self, file_in, size, buff_size=1024 * 64): part_num = 0 recv_left = size while 1: part_num += 1 read_size = min(buff_size, recv_left) part = file_in.read(read_size) if not part: break yield part if part_num % 100 == 0: # Avoid blocking ZeroNet execution during upload time.sleep(0.001) recv_left -= read_size if recv_left <= 0: break def hashBigfile(self, file_in, size, piece_size=1024 * 1024, file_out=None): self.site.settings["has_bigfile"] = True recv = 0 try: piece_hash = CryptHash.sha512t() piece_hashes = [] piece_recv = 0 mt = merkletools.MerkleTools() mt.hash_function = CryptHash.sha512t part = "" for part in self.readFile(file_in, size): if file_out: file_out.write(part) recv += len(part) piece_recv += len(part) piece_hash.update(part) if piece_recv >= piece_size: piece_digest = piece_hash.digest() piece_hashes.append(piece_digest) mt.leaves.append(piece_digest) piece_hash = CryptHash.sha512t() piece_recv = 0 if len(piece_hashes) % 100 == 0 or recv == size: self.log.info("- [HASHING:%.0f%%] Pieces: %s, %.1fMB/%.1fMB" % ( float(recv) / size * 100, len(piece_hashes), recv / 1024 / 1024, size / 1024 / 1024 )) part = "" if len(part) > 0: piece_digest = piece_hash.digest() piece_hashes.append(piece_digest) mt.leaves.append(piece_digest) except Exception as err: raise err finally: if file_out: file_out.close() mt.make_tree() return mt.get_merkle_root(), piece_size, { "sha512_pieces": piece_hashes } def hashFile(self, dir_inner_path, file_relative_path, optional=False): inner_path = dir_inner_path + file_relative_path file_size = self.site.storage.getSize(inner_path) # Only care about optional files >1MB if not optional or file_size < 1 * 1024 * 1024: return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional) back = {} content = self.contents.get(dir_inner_path + "content.json") hash = None piecemap_relative_path = None piece_size = None # Don't re-hash if it's already in content.json if content and file_relative_path in content.get("files_optional", {}): file_node = content["files_optional"][file_relative_path] if file_node["size"] == file_size: self.log.info("- [SAME SIZE] %s" % file_relative_path) hash = file_node.get("sha512") piecemap_relative_path = file_node.get("piecemap") piece_size = file_node.get("piece_size") if not hash or not piecemap_relative_path: # Not in content.json yet if file_size < 5 * 1024 * 1024: # Don't create piecemap automatically for files smaller than 5MB return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional) self.log.info("- [HASHING] %s" % file_relative_path) merkle_root, piece_size, piecemap_info = self.hashBigfile(self.site.storage.open(inner_path, "rb"), file_size) if not hash: hash = merkle_root if not piecemap_relative_path: file_name = helper.getFilename(file_relative_path) piecemap_relative_path = file_relative_path + ".piecemap.msgpack" piecemap_inner_path = inner_path + ".piecemap.msgpack" msgpack.pack({file_name: piecemap_info}, self.site.storage.open(piecemap_inner_path, "wb")) back.update(super(ContentManagerPlugin, self).hashFile(dir_inner_path, piecemap_relative_path, optional=True)) piece_num = int(math.ceil(float(file_size) / piece_size)) # Add the merkle root to hashfield hash_id = self.site.content_manager.hashfield.getHashId(hash) self.optionalDownloaded(inner_path, hash_id, file_size, own=True) self.site.storage.piecefields[hash].fromstring("1" * piece_num) back[file_relative_path] = {"sha512": hash, "size": file_size, "piecemap": piecemap_relative_path, "piece_size": piece_size} return back def getPiecemap(self, inner_path): file_info = self.site.content_manager.getFileInfo(inner_path) piecemap_inner_path = helper.getDirname(file_info["content_inner_path"]) + file_info["piecemap"] self.site.needFile(piecemap_inner_path, priority=20) piecemap = msgpack.unpack(self.site.storage.open(piecemap_inner_path))[helper.getFilename(inner_path)] piecemap["piece_size"] = file_info["piece_size"] return piecemap def verifyPiece(self, inner_path, pos, piece): piecemap = self.getPiecemap(inner_path) piece_i = int(pos / piecemap["piece_size"]) if CryptHash.sha512sum(piece, format="digest") != piecemap["sha512_pieces"][piece_i]: raise VerifyError("Invalid hash") return True def verifyFile(self, inner_path, file, ignore_same=True): if "|" not in inner_path: return super(ContentManagerPlugin, self).verifyFile(inner_path, file, ignore_same) inner_path, file_range = inner_path.split("|") pos_from, pos_to = map(int, file_range.split("-")) return self.verifyPiece(inner_path, pos_from, file) def optionalDownloaded(self, inner_path, hash_id, size=None, own=False): if "|" in inner_path: inner_path, file_range = inner_path.split("|") pos_from, pos_to = map(int, file_range.split("-")) file_info = self.getFileInfo(inner_path) # Mark piece downloaded piece_i = int(pos_from / file_info["piece_size"]) self.site.storage.piecefields[file_info["sha512"]][piece_i] = True # Only add to site size on first request if hash_id in self.hashfield: size = 0 elif size > 1024 * 1024: file_info = self.getFileInfo(inner_path) if file_info and "sha512" in file_info: # We already have the file, but not in piecefield sha512 = file_info["sha512"] if sha512 not in self.site.storage.piecefields: self.site.storage.checkBigfile(inner_path) return super(ContentManagerPlugin, self).optionalDownloaded(inner_path, hash_id, size, own) def optionalRemoved(self, inner_path, hash_id, size=None): if size and size > 1024 * 1024: file_info = self.getFileInfo(inner_path) sha512 = file_info["sha512"] if sha512 in self.site.storage.piecefields: del self.site.storage.piecefields[sha512] # Also remove other pieces of the file from download queue for key in list(self.site.bad_files.keys()): if key.startswith(inner_path + "|"): del self.site.bad_files[key] self.site.worker_manager.removeSolvedFileTasks() return super(ContentManagerPlugin, self).optionalRemoved(inner_path, hash_id, size) @PluginManager.registerTo("SiteStorage") class SiteStoragePlugin(object): def __init__(self, *args, **kwargs): super(SiteStoragePlugin, self).__init__(*args, **kwargs) self.piecefields = collections.defaultdict(BigfilePiecefield) if "piecefields" in self.site.settings.get("cache", {}): for sha512, piecefield_packed in self.site.settings["cache"].get("piecefields").items(): if piecefield_packed: self.piecefields[sha512].unpack(base64.b64decode(piecefield_packed)) self.site.settings["cache"]["piecefields"] = {} def createSparseFile(self, inner_path, size, sha512=None): file_path = self.getPath(inner_path) file_dir = os.path.dirname(file_path) if not os.path.isdir(file_dir): os.makedirs(file_dir) f = open(file_path, 'wb') f.truncate(min(1024 * 1024 * 5, size)) # Only pre-allocate up to 5MB f.close() if os.name == "nt": startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW subprocess.call(["fsutil", "sparse", "setflag", file_path], close_fds=True, startupinfo=startupinfo) if sha512 and sha512 in self.piecefields: self.log.debug("%s: File not exists, but has piecefield. Deleting piecefield." % inner_path) del self.piecefields[sha512] def write(self, inner_path, content): if "|" not in inner_path: return super(SiteStoragePlugin, self).write(inner_path, content) # Write to specific position by passing |{pos} after the filename inner_path, file_range = inner_path.split("|") pos_from, pos_to = map(int, file_range.split("-")) file_path = self.getPath(inner_path) # Create dir if not exist file_dir = os.path.dirname(file_path) if not os.path.isdir(file_dir): os.makedirs(file_dir) if not os.path.isfile(file_path): file_info = self.site.content_manager.getFileInfo(inner_path) self.createSparseFile(inner_path, file_info["size"]) # Write file with open(file_path, "rb+") as file: file.seek(pos_from) if hasattr(content, 'read'): # File-like object shutil.copyfileobj(content, file) # Write buff to disk else: # Simple string file.write(content) del content self.onUpdated(inner_path) def checkBigfile(self, inner_path): file_info = self.site.content_manager.getFileInfo(inner_path) if not file_info or (file_info and "piecemap" not in file_info): # It's not a big file return False self.site.settings["has_bigfile"] = True file_path = self.getPath(inner_path) sha512 = file_info["sha512"] piece_num = int(math.ceil(float(file_info["size"]) / file_info["piece_size"])) if os.path.isfile(file_path): if sha512 not in self.piecefields: if open(file_path).read(128) == "\0" * 128: piece_data = "0" else: piece_data = "1" self.log.debug("%s: File exists, but not in piecefield. Filling piecefiled with %s * %s." % (inner_path, piece_num, piece_data)) self.piecefields[sha512].fromstring(piece_data * piece_num) else: self.log.debug("Creating bigfile: %s" % inner_path) self.createSparseFile(inner_path, file_info["size"], sha512) self.piecefields[sha512].fromstring("0" * piece_num) return True def openBigfile(self, inner_path, prebuffer=0): if not self.checkBigfile(inner_path): return False self.site.needFile(inner_path, blocking=False) # Download piecemap return BigFile(self.site, inner_path, prebuffer=prebuffer) class BigFile(object): def __init__(self, site, inner_path, prebuffer=0): self.site = site self.inner_path = inner_path file_path = site.storage.getPath(inner_path) file_info = self.site.content_manager.getFileInfo(inner_path) self.piece_size = file_info["piece_size"] self.sha512 = file_info["sha512"] self.size = file_info["size"] self.prebuffer = prebuffer self.read_bytes = 0 self.piecefield = self.site.storage.piecefields[self.sha512] self.f = open(file_path, "rb+") self.read_lock = gevent.lock.Semaphore() def read(self, buff=64 * 1024): with self.read_lock: pos = self.f.tell() read_until = min(self.size, pos + buff) requests = [] # Request all required blocks while 1: piece_i = int(pos / self.piece_size) if piece_i * self.piece_size >= read_until: break pos_from = piece_i * self.piece_size pos_to = pos_from + self.piece_size if not self.piecefield[piece_i]: requests.append(self.site.needFile("%s|%s-%s" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=10)) pos += self.piece_size if not all(requests): return None # Request prebuffer if self.prebuffer: prebuffer_until = min(self.size, read_until + self.prebuffer) priority = 3 while 1: piece_i = int(pos / self.piece_size) if piece_i * self.piece_size >= prebuffer_until: break pos_from = piece_i * self.piece_size pos_to = pos_from + self.piece_size if not self.piecefield[piece_i]: self.site.needFile("%s|%s-%s" % (self.inner_path, pos_from, pos_to), blocking=False, update=True, priority=max(0, priority)) priority -= 1 pos += self.piece_size gevent.joinall(requests) self.read_bytes += buff # Increase buffer for long reads if self.read_bytes > 7 * 1024 * 1024 and self.prebuffer < 5 * 1024 * 1024: self.site.log.debug("%s: Increasing bigfile buffer size to 5MB..." % self.inner_path) self.prebuffer = 5 * 1024 * 1024 return self.f.read(buff) def seek(self, pos, whence=0): with self.read_lock: if whence == 2: # Relative from file end pos = self.size + pos # Use the real size instead of size on the disk whence = 0 return self.f.seek(pos, whence) def tell(self): return self.f.tell() def close(self): self.f.close() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() @PluginManager.registerTo("WorkerManager") class WorkerManagerPlugin(object): def addTask(self, inner_path, *args, **kwargs): file_info = kwargs.get("file_info") if file_info and "piecemap" in file_info: # Bigfile self.site.settings["has_bigfile"] = True piecemap_inner_path = helper.getDirname(file_info["content_inner_path"]) + file_info["piecemap"] piecemap_task = None if not self.site.storage.isFile(piecemap_inner_path): # Start download piecemap piecemap_task = super(WorkerManagerPlugin, self).addTask(piecemap_inner_path, priority=30) autodownload_bigfile_size_limit = self.site.settings.get("autodownload_bigfile_size_limit", config.autodownload_bigfile_size_limit) if "|" not in inner_path and self.site.isDownloadable(inner_path) and file_info["size"] / 1024 / 1024 <= autodownload_bigfile_size_limit: gevent.spawn_later(0.1, self.site.needFile, inner_path + "|all") # Download all pieces if "|" in inner_path: # Start download piece task = super(WorkerManagerPlugin, self).addTask(inner_path, *args, **kwargs) inner_path, file_range = inner_path.split("|") pos_from, pos_to = map(int, file_range.split("-")) task["piece_i"] = int(pos_from / file_info["piece_size"]) task["sha512"] = file_info["sha512"] else: if inner_path in self.site.bad_files: del self.site.bad_files[inner_path] if piecemap_task: task = piecemap_task else: fake_evt = gevent.event.AsyncResult() # Don't download anything if no range specified fake_evt.set(True) task = {"evt": fake_evt} if not self.site.storage.isFile(inner_path): self.site.storage.createSparseFile(inner_path, file_info["size"], file_info["sha512"]) piece_num = int(math.ceil(float(file_info["size"]) / file_info["piece_size"])) self.site.storage.piecefields[file_info["sha512"]].fromstring("0" * piece_num) else: task = super(WorkerManagerPlugin, self).addTask(inner_path, *args, **kwargs) return task def taskAddPeer(self, task, peer): if "piece_i" in task: if not peer.piecefields[task["sha512"]][task["piece_i"]]: if task["sha512"] not in peer.piecefields: gevent.spawn(peer.updatePiecefields, force=True) elif not task["peers"]: gevent.spawn(peer.updatePiecefields) return False # Deny to add peers to task if file not in piecefield return super(WorkerManagerPlugin, self).taskAddPeer(task, peer) @PluginManager.registerTo("FileRequest") class FileRequestPlugin(object): def isReadable(self, site, inner_path, file, pos): # Peek into file if file.read(10) == b"\0" * 10: # Looks empty, but makes sures we don't have that piece file_info = site.content_manager.getFileInfo(inner_path) piece_i = int(pos / file_info["piece_size"]) if not site.storage.piecefields[file_info["sha512"]][piece_i]: return False # Seek back to position we want to read file.seek(pos) return super(FileRequestPlugin, self).isReadable(site, inner_path, file, pos) def actionGetPiecefields(self, params): site = self.sites.get(params["site"]) if not site or not site.settings["serving"]: # Site unknown or not serving self.response({"error": "Unknown site"}) return False # Add peer to site if not added before peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True) if not peer.connection: # Just added peer.connect(self.connection) # Assign current connection to peer piecefields_packed = {sha512: piecefield.pack() for sha512, piecefield in site.storage.piecefields.items()} self.response({"piecefields_packed": piecefields_packed}) def actionSetPiecefields(self, params): site = self.sites.get(params["site"]) if not site or not site.settings["serving"]: # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False # Add or get peer peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, connection=self.connection) if not peer.connection: peer.connect(self.connection) peer.piecefields = collections.defaultdict(BigfilePiecefieldPacked) for sha512, piecefield_packed in params["piecefields_packed"].items(): peer.piecefields[sha512].unpack(piecefield_packed) site.settings["has_bigfile"] = True self.response({"ok": "Updated"}) @PluginManager.registerTo("Peer") class PeerPlugin(object): def __getattr__(self, key): if key == "piecefields": self.piecefields = collections.defaultdict(BigfilePiecefieldPacked) return self.piecefields elif key == "time_piecefields_updated": self.time_piecefields_updated = None return self.time_piecefields_updated else: return super(PeerPlugin, self).__getattr__(key) @util.Noparallel(ignore_args=True) def updatePiecefields(self, force=False): if self.connection and self.connection.handshake.get("rev", 0) < 2190: return False # Not supported # Don't update piecefield again in 1 min if self.time_piecefields_updated and time.time() - self.time_piecefields_updated < 60 and not force: return False self.time_piecefields_updated = time.time() res = self.request("getPiecefields", {"site": self.site.address}) if not res or "error" in res: return False self.piecefields = collections.defaultdict(BigfilePiecefieldPacked) try: for sha512, piecefield_packed in res["piecefields_packed"].items(): self.piecefields[sha512].unpack(piecefield_packed) except Exception as err: self.log("Invalid updatePiecefields response: %s" % Debug.formatException(err)) return self.piecefields def sendMyHashfield(self, *args, **kwargs): return super(PeerPlugin, self).sendMyHashfield(*args, **kwargs) def updateHashfield(self, *args, **kwargs): if self.site.settings.get("has_bigfile"): thread = gevent.spawn(self.updatePiecefields, *args, **kwargs) back = super(PeerPlugin, self).updateHashfield(*args, **kwargs) thread.join() return back else: return super(PeerPlugin, self).updateHashfield(*args, **kwargs) def getFile(self, site, inner_path, *args, **kwargs): if "|" in inner_path: inner_path, file_range = inner_path.split("|") pos_from, pos_to = map(int, file_range.split("-")) kwargs["pos_from"] = pos_from kwargs["pos_to"] = pos_to return super(PeerPlugin, self).getFile(site, inner_path, *args, **kwargs) @PluginManager.registerTo("Site") class SitePlugin(object): def isFileDownloadAllowed(self, inner_path, file_info): if "piecemap" in file_info: file_size_mb = file_info["size"] / 1024 / 1024 if config.bigfile_size_limit and file_size_mb > config.bigfile_size_limit: self.log.debug( "Bigfile size %s too large: %sMB > %sMB, skipping..." % (inner_path, file_size_mb, config.bigfile_size_limit) ) return False file_info = file_info.copy() file_info["size"] = file_info["piece_size"] return super(SitePlugin, self).isFileDownloadAllowed(inner_path, file_info) def getSettingsCache(self): back = super(SitePlugin, self).getSettingsCache() if self.storage.piecefields: back["piecefields"] = {sha512: base64.b64encode(piecefield.pack()).decode("utf8") for sha512, piecefield in self.storage.piecefields.items()} return back def needFile(self, inner_path, *args, **kwargs): if inner_path.endswith("|all"): @util.Pooled(20) def pooledNeedBigfile(inner_path, *args, **kwargs): if inner_path not in self.bad_files: self.log.debug("Cancelled piece, skipping %s" % inner_path) return False return self.needFile(inner_path, *args, **kwargs) inner_path = inner_path.replace("|all", "") file_info = self.needFileInfo(inner_path) file_size = file_info["size"] piece_size = file_info["piece_size"] piece_num = int(math.ceil(float(file_size) / piece_size)) file_threads = [] piecefield = self.storage.piecefields.get(file_info["sha512"]) for piece_i in range(piece_num): piece_from = piece_i * piece_size piece_to = min(file_size, piece_from + piece_size) if not piecefield or not piecefield[piece_i]: inner_path_piece = "%s|%s-%s" % (inner_path, piece_from, piece_to) self.bad_files[inner_path_piece] = self.bad_files.get(inner_path_piece, 1) res = pooledNeedBigfile(inner_path_piece, blocking=False) if res is not True and res is not False: file_threads.append(res) gevent.joinall(file_threads) else: return super(SitePlugin, self).needFile(inner_path, *args, **kwargs) @PluginManager.registerTo("ConfigPlugin") class ConfigPlugin(object): def createArguments(self): group = self.parser.add_argument_group("Bigfile plugin") group.add_argument('--autodownload_bigfile_size_limit', help='Also download bigfiles smaller than this limit if help distribute option is checked', default=1, metavar="MB", type=int) group.add_argument('--bigfile_size_limit', help='Maximum size of downloaded big files', default=False, metavar="MB", type=int) return super(ConfigPlugin, self).createArguments()