diff --git a/plugins/FilePack/FilePackPlugin.py b/plugins/FilePack/FilePackPlugin.py index 8e3bc94e..20c290f3 100644 --- a/plugins/FilePack/FilePackPlugin.py +++ b/plugins/FilePack/FilePackPlugin.py @@ -5,6 +5,7 @@ import gevent from Plugin import PluginManager from Config import config +from Debug import Debug # Keep archive open for faster reponse times for large sites @@ -16,24 +17,25 @@ def closeArchive(archive_path): del archive_cache[archive_path] -def openArchive(archive_path): +def openArchive(archive_path, file_obj=None): if archive_path not in archive_cache: if archive_path.endswith("tar.gz"): import tarfile - archive_cache[archive_path] = tarfile.open(archive_path, "r:gz") + archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:gz") elif archive_path.endswith("tar.bz2"): import tarfile - archive_cache[archive_path] = tarfile.open(archive_path, "r:bz2") + archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:bz2") else: import zipfile - archive_cache[archive_path] = zipfile.ZipFile(archive_path) + archive_cache[archive_path] = zipfile.ZipFile(file_obj or archive_path) gevent.spawn_later(5, lambda: closeArchive(archive_path)) # Close after 5 sec archive = archive_cache[archive_path] return archive -def openArchiveFile(archive_path, path_within): - archive = openArchive(archive_path) + +def openArchiveFile(archive_path, path_within, file_obj=None): + archive = openArchive(archive_path, file_obj=file_obj) if archive_path.endswith(".zip"): return archive.open(path_within) else: @@ -44,20 +46,24 @@ def openArchiveFile(archive_path, path_within): class UiRequestPlugin(object): def actionSiteMedia(self, path, **kwargs): if ".zip/" in path or ".tar.gz/" in path: + file_obj = None path_parts = self.parsePath(path) file_path = u"%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"].decode("utf8")) match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path) archive_path, path_within = match.groups() - if not os.path.isfile(archive_path): + if archive_path not in archive_cache: site = self.server.site_manager.get(path_parts["address"]) if not site: return self.actionSiteAddPrompt(path) - # Wait until file downloads - result = site.needFile(site.storage.getInnerPath(archive_path), priority=10) - # Send virutal file path download finished event to remove loading screen - site.updateWebsocket(file_done=site.storage.getInnerPath(file_path)) - if not result: - return self.error404(path) + archive_inner_path = site.storage.getInnerPath(archive_path) + if not os.path.isfile(archive_path): + # Wait until file downloads + result = site.needFile(archive_inner_path, priority=10) + # Send virutal file path download finished event to remove loading screen + site.updateWebsocket(file_done=archive_inner_path) + if not result: + return self.error404(archive_inner_path) + file_obj = site.storage.openBigfile(archive_inner_path) header_allow_ajax = False if self.get.get("ajax_key"): @@ -68,12 +74,12 @@ class UiRequestPlugin(object): return self.error403("Invalid ajax_key") try: - file = openArchiveFile(archive_path, path_within) + file = openArchiveFile(archive_path, path_within, file_obj=file_obj) content_type = self.getContentType(file_path) self.sendHeader(200, content_type=content_type, noscript=kwargs.get("header_noscript", False), allow_ajax=header_allow_ajax) return self.streamFile(file) except Exception as err: - self.log.debug("Error opening archive file: %s" % err) + self.log.debug("Error opening archive file: %s" % Debug.formatException(err)) return self.error404(path) return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs) @@ -105,7 +111,21 @@ class SiteStoragePlugin(object): if ".zip" in inner_path or ".tar.gz" in inner_path: match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path) archive_inner_path, path_within = match.groups() - archive = openArchive(self.getPath(archive_inner_path)) + archive_path = self.getPath(archive_inner_path) + file_obj = None + if archive_path not in archive_cache: + if not os.path.isfile(archive_path): + result = self.site.needFile(archive_inner_path, priority=10) + self.site.updateWebsocket(file_done=archive_inner_path) + if not result: + raise Exception("Unable to download file") + file_obj = self.site.storage.openBigfile(archive_inner_path) + + try: + archive = openArchive(archive_path, file_obj=file_obj) + except Exception as err: + raise Exception("Unable to download file: %s" % err) + if archive_inner_path.endswith(".zip"): namelist = [name for name in archive.namelist() if not name.endswith("/")] else: