195 lines
7.7 KiB
Python
195 lines
7.7 KiB
Python
import os
|
|
import re
|
|
|
|
import gevent
|
|
|
|
from Plugin import PluginManager
|
|
from Config import config
|
|
from Debug import Debug
|
|
|
|
|
|
# Keep archive open for faster reponse times for large sites
|
|
archive_cache = {}
|
|
|
|
|
|
def closeArchive(archive_path):
|
|
if archive_path in archive_cache:
|
|
del archive_cache[archive_path]
|
|
|
|
|
|
def openArchive(archive_path, file_obj=None):
|
|
if archive_path not in archive_cache:
|
|
if archive_path.endswith("tar.gz"):
|
|
import tarfile
|
|
archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:gz")
|
|
elif archive_path.endswith("tar.bz2"):
|
|
import tarfile
|
|
archive_cache[archive_path] = tarfile.open(file_obj or archive_path, "r:bz2")
|
|
else:
|
|
import zipfile
|
|
archive_cache[archive_path] = zipfile.ZipFile(file_obj or archive_path)
|
|
gevent.spawn_later(5, lambda: closeArchive(archive_path)) # Close after 5 sec
|
|
|
|
archive = archive_cache[archive_path]
|
|
return archive
|
|
|
|
|
|
def openArchiveFile(archive_path, path_within, file_obj=None):
|
|
archive = openArchive(archive_path, file_obj=file_obj)
|
|
if archive_path.endswith(".zip"):
|
|
return archive.open(path_within)
|
|
else:
|
|
return archive.extractfile(path_within.encode("utf8"))
|
|
|
|
|
|
@PluginManager.registerTo("UiRequest")
|
|
class UiRequestPlugin(object):
|
|
def actionSiteMedia(self, path, **kwargs):
|
|
if ".zip/" in path or ".tar.gz/" in path:
|
|
file_obj = None
|
|
path_parts = self.parsePath(path)
|
|
file_path = "%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"])
|
|
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path)
|
|
archive_path, path_within = match.groups()
|
|
if archive_path not in archive_cache:
|
|
site = self.server.site_manager.get(path_parts["address"])
|
|
if not site:
|
|
return self.actionSiteAddPrompt(path)
|
|
archive_inner_path = site.storage.getInnerPath(archive_path)
|
|
if not os.path.isfile(archive_path):
|
|
# Wait until file downloads
|
|
result = site.needFile(archive_inner_path, priority=10)
|
|
# Send virutal file path download finished event to remove loading screen
|
|
site.updateWebsocket(file_done=archive_inner_path)
|
|
if not result:
|
|
return self.error404(archive_inner_path)
|
|
file_obj = site.storage.openBigfile(archive_inner_path)
|
|
|
|
header_allow_ajax = False
|
|
if self.get.get("ajax_key"):
|
|
requester_site = self.server.site_manager.get(path_parts["request_address"])
|
|
if self.get["ajax_key"] == requester_site.settings["ajax_key"]:
|
|
header_allow_ajax = True
|
|
else:
|
|
return self.error403("Invalid ajax_key")
|
|
|
|
try:
|
|
file = openArchiveFile(archive_path, path_within, file_obj=file_obj)
|
|
content_type = self.getContentType(file_path)
|
|
self.sendHeader(200, content_type=content_type, noscript=kwargs.get("header_noscript", False), allow_ajax=header_allow_ajax)
|
|
return self.streamFile(file)
|
|
except Exception as err:
|
|
self.log.debug("Error opening archive file: %s" % Debug.formatException(err))
|
|
return self.error404(path)
|
|
|
|
return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs)
|
|
|
|
def streamFile(self, file):
|
|
for i in range(100): # Read max 6MB
|
|
try:
|
|
block = file.read(60 * 1024)
|
|
if block:
|
|
yield block
|
|
else:
|
|
raise StopIteration
|
|
except StopIteration:
|
|
file.close()
|
|
break
|
|
|
|
|
|
@PluginManager.registerTo("SiteStorage")
|
|
class SiteStoragePlugin(object):
|
|
def isFile(self, inner_path):
|
|
if ".zip/" in inner_path or ".tar.gz/" in inner_path:
|
|
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", inner_path)
|
|
archive_inner_path, path_within = match.groups()
|
|
return super(SiteStoragePlugin, self).isFile(archive_inner_path)
|
|
else:
|
|
return super(SiteStoragePlugin, self).isFile(inner_path)
|
|
|
|
def openArchive(self, inner_path):
|
|
archive_path = self.getPath(inner_path)
|
|
file_obj = None
|
|
if archive_path not in archive_cache:
|
|
if not os.path.isfile(archive_path):
|
|
result = self.site.needFile(inner_path, priority=10)
|
|
self.site.updateWebsocket(file_done=inner_path)
|
|
if not result:
|
|
raise Exception("Unable to download file")
|
|
file_obj = self.site.storage.openBigfile(inner_path)
|
|
|
|
try:
|
|
archive = openArchive(archive_path, file_obj=file_obj)
|
|
except Exception as err:
|
|
raise Exception("Unable to download file: %s" % err)
|
|
|
|
return archive
|
|
|
|
def walk(self, inner_path, *args, **kwags):
|
|
if ".zip" in inner_path or ".tar.gz" in inner_path:
|
|
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
|
|
archive_inner_path, path_within = match.groups()
|
|
archive = self.openArchive(archive_inner_path)
|
|
path_within = path_within.lstrip("/")
|
|
|
|
if archive_inner_path.endswith(".zip"):
|
|
namelist = [name for name in archive.namelist() if not name.endswith("/")]
|
|
else:
|
|
namelist = [item.name for item in archive.getmembers() if not item.isdir()]
|
|
|
|
namelist_relative = []
|
|
for name in namelist:
|
|
if not name.startswith(path_within):
|
|
continue
|
|
name_relative = name.replace(path_within, "", 1).rstrip("/")
|
|
namelist_relative.append(name_relative)
|
|
|
|
return namelist_relative
|
|
|
|
else:
|
|
return super(SiteStoragePlugin, self).walk(inner_path, *args, **kwags)
|
|
|
|
def list(self, inner_path, *args, **kwags):
|
|
if ".zip" in inner_path or ".tar.gz" in inner_path:
|
|
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
|
|
archive_inner_path, path_within = match.groups()
|
|
archive = self.openArchive(archive_inner_path)
|
|
path_within = path_within.lstrip("/")
|
|
|
|
if archive_inner_path.endswith(".zip"):
|
|
namelist = [name for name in archive.namelist()]
|
|
else:
|
|
namelist = [item.name for item in archive.getmembers()]
|
|
|
|
namelist_relative = []
|
|
for name in namelist:
|
|
if not name.startswith(path_within):
|
|
continue
|
|
name_relative = name.replace(path_within, "", 1).rstrip("/")
|
|
|
|
if "/" in name_relative: # File is in sub-directory
|
|
continue
|
|
|
|
namelist_relative.append(name_relative)
|
|
return namelist_relative
|
|
|
|
else:
|
|
return super(SiteStoragePlugin, self).list(inner_path, *args, **kwags)
|
|
|
|
def read(self, inner_path, mode="r"):
|
|
if ".zip/" in inner_path or ".tar.gz/" in inner_path:
|
|
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))(.*)", inner_path)
|
|
archive_inner_path, path_within = match.groups()
|
|
archive = self.openArchive(archive_inner_path)
|
|
path_within = path_within.lstrip("/")
|
|
print archive, archive_inner_path
|
|
|
|
if archive_inner_path.endswith(".zip"):
|
|
return archive.open(path_within).read()
|
|
else:
|
|
return archive.extractfile(path_within.encode("utf8")).read()
|
|
|
|
else:
|
|
return super(SiteStoragePlugin, self).read(inner_path, mode)
|
|
|