2015-07-12 20:36:46 +02:00
|
|
|
import os
|
|
|
|
import re
|
|
|
|
import shutil
|
|
|
|
import json
|
|
|
|
import time
|
2015-11-15 11:13:57 +01:00
|
|
|
import sys
|
2018-03-29 02:49:06 +02:00
|
|
|
from collections import defaultdict
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
import sqlite3
|
2015-03-19 21:19:14 +01:00
|
|
|
import gevent.event
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-12-20 23:35:49 +01:00
|
|
|
import util
|
|
|
|
from util import SafeRe
|
2015-03-19 21:19:14 +01:00
|
|
|
from Db import Db
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
from Debug import Debug
|
2015-05-31 15:52:21 +02:00
|
|
|
from Config import config
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
from util import helper
|
2016-08-10 12:28:57 +02:00
|
|
|
from Plugin import PluginManager
|
2017-05-07 18:14:13 +02:00
|
|
|
from Translate import translate as _
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
|
2015-03-19 21:19:14 +01:00
|
|
|
|
2016-08-10 12:28:57 +02:00
|
|
|
@PluginManager.acceptPlugins
|
|
|
|
class SiteStorage(object):
|
2015-07-12 20:36:46 +02:00
|
|
|
def __init__(self, site, allow_create=True):
|
|
|
|
self.site = site
|
2019-03-15 21:06:59 +01:00
|
|
|
self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory
|
2017-01-22 21:22:53 +01:00
|
|
|
self.allowed_dir = os.path.abspath(self.directory) # Only serve file within this dir
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log = site.log
|
|
|
|
self.db = None # Db class
|
|
|
|
self.db_checked = False # Checked db tables since startup
|
|
|
|
self.event_db_busy = None # Gevent AsyncResult if db is working on rebuild
|
|
|
|
self.has_db = self.isFile("dbschema.json") # The site has schema
|
|
|
|
|
|
|
|
if not os.path.isdir(self.directory):
|
|
|
|
if allow_create:
|
|
|
|
os.mkdir(self.directory) # Create directory if not found
|
|
|
|
else:
|
|
|
|
raise Exception("Directory not exists: %s" % self.directory)
|
|
|
|
|
2017-10-22 11:13:09 +02:00
|
|
|
def getDbFile(self):
|
|
|
|
if self.isFile("dbschema.json"):
|
|
|
|
schema = self.loadJson("dbschema.json")
|
|
|
|
return schema["db_file"]
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
|
2018-01-19 02:31:46 +01:00
|
|
|
# Create new databaseobject with the site's schema
|
2018-11-08 01:20:33 +01:00
|
|
|
def openDb(self, close_idle=False):
|
2018-01-19 02:31:46 +01:00
|
|
|
schema = self.getDbSchema()
|
|
|
|
db_path = self.getPath(schema["db_file"])
|
2018-11-08 01:20:33 +01:00
|
|
|
return Db(schema, db_path, close_idle=close_idle)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
def closeDb(self):
|
|
|
|
if self.db:
|
|
|
|
self.db.close()
|
|
|
|
self.event_db_busy = None
|
|
|
|
self.db = None
|
|
|
|
|
2018-01-19 02:31:46 +01:00
|
|
|
def getDbSchema(self):
|
|
|
|
try:
|
|
|
|
schema = self.loadJson("dbschema.json")
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2018-01-19 02:31:46 +01:00
|
|
|
raise Exception("dbschema.json is not a valid JSON: %s" % err)
|
|
|
|
return schema
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Return db class
|
|
|
|
def getDb(self):
|
|
|
|
if not self.db:
|
|
|
|
self.log.debug("No database, waiting for dbschema.json...")
|
2015-09-27 02:08:53 +02:00
|
|
|
self.site.needFile("dbschema.json", priority=3)
|
2015-07-12 20:36:46 +02:00
|
|
|
self.has_db = self.isFile("dbschema.json") # Recheck if dbschema exist
|
|
|
|
if self.has_db:
|
2018-01-19 02:31:46 +01:00
|
|
|
schema = self.getDbSchema()
|
|
|
|
db_path = self.getPath(schema["db_file"])
|
|
|
|
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
|
|
|
|
self.rebuildDb()
|
|
|
|
|
|
|
|
if self.db:
|
|
|
|
self.db.close()
|
2018-11-08 01:20:33 +01:00
|
|
|
self.db = self.openDb(close_idle=True)
|
2018-01-19 02:31:46 +01:00
|
|
|
|
|
|
|
changed_tables = self.db.checkTables()
|
|
|
|
if changed_tables:
|
|
|
|
self.rebuildDb(delete_db=False) # TODO: only update the changed table datas
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
return self.db
|
|
|
|
|
2017-02-09 01:56:20 +01:00
|
|
|
def updateDbFile(self, inner_path, file=None, cur=None):
|
|
|
|
path = self.getPath(inner_path)
|
|
|
|
return self.getDb().updateJson(path, file, cur)
|
|
|
|
|
2016-08-10 12:30:04 +02:00
|
|
|
# Return possible db files for the site
|
|
|
|
def getDbFiles(self):
|
2017-12-02 02:38:17 +01:00
|
|
|
found = 0
|
2019-03-15 21:06:59 +01:00
|
|
|
for content_inner_path, content in self.site.content_manager.contents.items():
|
2016-08-10 12:30:04 +02:00
|
|
|
# content.json file itself
|
2017-03-06 15:30:42 +01:00
|
|
|
if self.isFile(content_inner_path):
|
2017-05-07 18:14:13 +02:00
|
|
|
yield content_inner_path, self.getPath(content_inner_path)
|
2016-08-10 12:30:04 +02:00
|
|
|
else:
|
|
|
|
self.log.error("[MISSING] %s" % content_inner_path)
|
|
|
|
# Data files in content.json
|
|
|
|
content_inner_path_dir = helper.getDirname(content_inner_path) # Content.json dir relative to site
|
2019-03-15 21:06:59 +01:00
|
|
|
for file_relative_path in list(content.get("files", {}).keys()) + list(content.get("files_optional", {}).keys()):
|
2017-08-09 14:19:39 +02:00
|
|
|
if not file_relative_path.endswith(".json") and not file_relative_path.endswith("json.gz"):
|
2016-08-10 12:30:04 +02:00
|
|
|
continue # We only interesed in json files
|
|
|
|
file_inner_path = content_inner_path_dir + file_relative_path # File Relative to site dir
|
|
|
|
file_inner_path = file_inner_path.strip("/") # Strip leading /
|
|
|
|
if self.isFile(file_inner_path):
|
2017-05-07 18:14:13 +02:00
|
|
|
yield file_inner_path, self.getPath(file_inner_path)
|
2016-08-10 12:30:04 +02:00
|
|
|
else:
|
|
|
|
self.log.error("[MISSING] %s" % file_inner_path)
|
2017-12-02 02:38:17 +01:00
|
|
|
found += 1
|
|
|
|
if found % 100 == 0:
|
2019-03-16 00:10:49 +01:00
|
|
|
time.sleep(0.001) # Context switch to avoid UI block
|
2016-08-10 12:30:04 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Rebuild sql cache
|
2018-09-02 02:24:07 +02:00
|
|
|
@util.Noparallel()
|
2015-07-12 20:36:46 +02:00
|
|
|
def rebuildDb(self, delete_db=True):
|
|
|
|
self.has_db = self.isFile("dbschema.json")
|
|
|
|
if not self.has_db:
|
|
|
|
return False
|
|
|
|
self.event_db_busy = gevent.event.AsyncResult()
|
|
|
|
schema = self.loadJson("dbschema.json")
|
|
|
|
db_path = self.getPath(schema["db_file"])
|
|
|
|
if os.path.isfile(db_path) and delete_db:
|
|
|
|
if self.db:
|
|
|
|
self.db.close() # Close db if open
|
2016-08-10 12:30:04 +02:00
|
|
|
time.sleep(0.5)
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.info("Deleting %s" % db_path)
|
|
|
|
try:
|
|
|
|
os.unlink(db_path)
|
2019-03-16 00:10:49 +01:00
|
|
|
except Exception as err:
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.error("Delete error: %s" % err)
|
2018-01-19 02:31:46 +01:00
|
|
|
|
|
|
|
db = self.openDb()
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.info("Creating tables...")
|
2018-01-19 02:31:46 +01:00
|
|
|
db.checkTables()
|
|
|
|
cur = db.getCursor()
|
2015-07-12 20:36:46 +02:00
|
|
|
cur.execute("BEGIN")
|
|
|
|
cur.logging = False
|
|
|
|
found = 0
|
|
|
|
s = time.time()
|
2017-12-02 02:38:17 +01:00
|
|
|
self.log.info("Getting db files...")
|
2017-05-07 18:14:13 +02:00
|
|
|
db_files = list(self.getDbFiles())
|
2017-12-02 02:38:17 +01:00
|
|
|
self.log.info("Importing data...")
|
2016-08-10 12:30:04 +02:00
|
|
|
try:
|
2017-05-07 18:14:13 +02:00
|
|
|
if len(db_files) > 100:
|
|
|
|
self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files..."].format("0000", len(db_files)), "rebuild", 0)
|
|
|
|
for file_inner_path, file_path in db_files:
|
2016-08-27 11:01:41 +02:00
|
|
|
try:
|
2017-05-07 18:14:13 +02:00
|
|
|
if self.updateDbFile(file_inner_path, file=open(file_path, "rb"), cur=cur):
|
2016-08-27 11:01:41 +02:00
|
|
|
found += 1
|
|
|
|
except Exception, err:
|
|
|
|
self.log.error("Error importing %s: %s" % (file_inner_path, Debug.formatException(err)))
|
2017-05-07 18:14:13 +02:00
|
|
|
if found and found % 100 == 0:
|
|
|
|
self.site.messageWebsocket(
|
|
|
|
_["Database rebuilding...<br>Imported {0} of {1} files..."].format(found, len(db_files)),
|
|
|
|
"rebuild",
|
|
|
|
int(float(found) / len(db_files) * 100)
|
|
|
|
)
|
2017-12-02 02:38:17 +01:00
|
|
|
time.sleep(0.000001) # Context switch to avoid UI block
|
2016-08-27 11:01:41 +02:00
|
|
|
|
2016-08-10 12:30:04 +02:00
|
|
|
finally:
|
|
|
|
cur.execute("END")
|
2018-01-19 02:31:46 +01:00
|
|
|
cur.close()
|
|
|
|
db.close()
|
|
|
|
self.log.info("Closing Db: %s" % db)
|
2017-05-07 18:14:13 +02:00
|
|
|
if len(db_files) > 100:
|
|
|
|
self.site.messageWebsocket(_["Database rebuilding...<br>Imported {0} of {1} files..."].format(found, len(db_files)), "rebuild", 100)
|
2016-08-10 12:30:04 +02:00
|
|
|
self.log.info("Imported %s data file in %ss" % (found, time.time() - s))
|
|
|
|
self.event_db_busy.set(True) # Event done, notify waiters
|
|
|
|
self.event_db_busy = None # Clear event
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Execute sql query or rebuild on dberror
|
|
|
|
def query(self, query, params=None):
|
2018-12-15 17:45:17 +01:00
|
|
|
if not query.strip().upper().startswith("SELECT"):
|
|
|
|
raise Exception("Only SELECT query supported")
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if self.event_db_busy: # Db not ready for queries
|
|
|
|
self.log.debug("Wating for db...")
|
|
|
|
self.event_db_busy.get() # Wait for event
|
|
|
|
try:
|
|
|
|
res = self.getDb().execute(query, params)
|
2019-03-15 21:06:59 +01:00
|
|
|
except sqlite3.DatabaseError as err:
|
2015-07-12 20:36:46 +02:00
|
|
|
if err.__class__.__name__ == "DatabaseError":
|
|
|
|
self.log.error("Database error: %s, query: %s, try to rebuilding it..." % (err, query))
|
|
|
|
self.rebuildDb()
|
|
|
|
res = self.db.cur.execute(query, params)
|
|
|
|
else:
|
|
|
|
raise err
|
|
|
|
return res
|
|
|
|
|
|
|
|
# Open file object
|
2019-03-16 00:10:49 +01:00
|
|
|
def open(self, inner_path, mode="rb", create_dirs=False, **kwargs):
|
2017-08-18 14:43:28 +02:00
|
|
|
file_path = self.getPath(inner_path)
|
|
|
|
if create_dirs:
|
|
|
|
file_dir = os.path.dirname(file_path)
|
|
|
|
if not os.path.isdir(file_dir):
|
|
|
|
os.makedirs(file_dir)
|
2019-03-16 00:10:49 +01:00
|
|
|
return open(file_path, mode, **kwargs)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Open file object
|
2019-03-16 00:10:49 +01:00
|
|
|
def read(self, inner_path, mode="rb"):
|
2015-07-12 20:36:46 +02:00
|
|
|
return open(self.getPath(inner_path), mode).read()
|
|
|
|
|
|
|
|
# Write content to file
|
|
|
|
def write(self, inner_path, content):
|
|
|
|
file_path = self.getPath(inner_path)
|
|
|
|
# Create dir if not exist
|
|
|
|
file_dir = os.path.dirname(file_path)
|
|
|
|
if not os.path.isdir(file_dir):
|
|
|
|
os.makedirs(file_dir)
|
|
|
|
# Write file
|
|
|
|
if hasattr(content, 'read'): # File-like object
|
|
|
|
with open(file_path, "wb") as file:
|
|
|
|
shutil.copyfileobj(content, file) # Write buff to disk
|
|
|
|
else: # Simple string
|
2017-01-05 02:26:44 +01:00
|
|
|
if inner_path == "content.json" and os.path.isfile(file_path):
|
|
|
|
helper.atomicWrite(file_path, content)
|
|
|
|
else:
|
|
|
|
with open(file_path, "wb") as file:
|
|
|
|
file.write(content)
|
2015-07-12 20:36:46 +02:00
|
|
|
del content
|
|
|
|
self.onUpdated(inner_path)
|
|
|
|
|
2015-09-16 00:01:23 +02:00
|
|
|
# Remove file from filesystem
|
|
|
|
def delete(self, inner_path):
|
|
|
|
file_path = self.getPath(inner_path)
|
|
|
|
os.unlink(file_path)
|
2016-09-14 10:52:41 +02:00
|
|
|
self.onUpdated(inner_path, file=False)
|
2015-09-16 00:01:23 +02:00
|
|
|
|
2016-03-26 00:22:27 +01:00
|
|
|
def deleteDir(self, inner_path):
|
|
|
|
dir_path = self.getPath(inner_path)
|
|
|
|
os.rmdir(dir_path)
|
|
|
|
|
2016-04-06 13:50:20 +02:00
|
|
|
def rename(self, inner_path_before, inner_path_after):
|
2016-04-07 10:36:50 +02:00
|
|
|
for retry in range(3):
|
|
|
|
# To workaround "The process cannot access the file beacause it is being used by another process." error
|
|
|
|
try:
|
|
|
|
os.rename(self.getPath(inner_path_before), self.getPath(inner_path_after))
|
|
|
|
err = None
|
|
|
|
break
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2016-04-09 19:43:44 +02:00
|
|
|
self.log.error("%s rename error: %s (retry #%s)" % (inner_path_before, err, retry))
|
2016-04-07 10:36:50 +02:00
|
|
|
time.sleep(0.1 + retry)
|
|
|
|
if err:
|
|
|
|
raise err
|
2016-04-06 13:50:20 +02:00
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
# List files from a directory
|
2017-12-20 23:25:25 +01:00
|
|
|
def walk(self, dir_inner_path, ignore=None):
|
2015-09-28 22:07:26 +02:00
|
|
|
directory = self.getPath(dir_inner_path)
|
|
|
|
for root, dirs, files in os.walk(directory):
|
|
|
|
root = root.replace("\\", "/")
|
|
|
|
root_relative_path = re.sub("^%s" % re.escape(directory), "", root).lstrip("/")
|
|
|
|
for file_name in files:
|
|
|
|
if root_relative_path: # Not root dir
|
2017-12-20 23:25:25 +01:00
|
|
|
file_relative_path = root_relative_path + "/" + file_name
|
2015-09-28 22:07:26 +02:00
|
|
|
else:
|
2017-12-20 23:25:25 +01:00
|
|
|
file_relative_path = file_name
|
|
|
|
|
|
|
|
if ignore and SafeRe.match(ignore, file_relative_path):
|
|
|
|
continue
|
|
|
|
|
|
|
|
yield file_relative_path
|
|
|
|
|
|
|
|
# Don't scan directory that is in the ignore pattern
|
|
|
|
if ignore:
|
|
|
|
dirs_filtered = []
|
|
|
|
for dir_name in dirs:
|
|
|
|
if root_relative_path:
|
|
|
|
dir_relative_path = root_relative_path + "/" + dir_name
|
|
|
|
else:
|
|
|
|
dir_relative_path = dir_name
|
|
|
|
|
|
|
|
if ignore == ".*" or re.match(".*([|(]|^)%s([|)]|$)" % re.escape(dir_relative_path + "/.*"), ignore):
|
|
|
|
continue
|
|
|
|
|
|
|
|
dirs_filtered.append(dir_name)
|
|
|
|
dirs[:] = dirs_filtered
|
2015-09-28 22:07:26 +02:00
|
|
|
|
2017-02-25 05:47:38 +01:00
|
|
|
# list directories in a directory
|
|
|
|
def list(self, dir_inner_path):
|
|
|
|
directory = self.getPath(dir_inner_path)
|
|
|
|
return os.listdir(directory)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Site content updated
|
2016-08-10 12:31:30 +02:00
|
|
|
def onUpdated(self, inner_path, file=None):
|
2015-07-12 20:36:46 +02:00
|
|
|
# Update Sql cache
|
|
|
|
if inner_path == "dbschema.json":
|
|
|
|
self.has_db = self.isFile("dbschema.json")
|
2015-11-05 23:19:36 +01:00
|
|
|
# Reopen DB to check changes
|
2016-08-10 12:31:30 +02:00
|
|
|
if self.has_db:
|
|
|
|
self.closeDb()
|
2018-01-19 02:31:46 +01:00
|
|
|
self.getDb()
|
2017-08-09 14:19:39 +02:00
|
|
|
elif not config.disable_db and (inner_path.endswith(".json") or inner_path.endswith(".json.gz")) and self.has_db: # Load json file to db
|
2016-04-06 13:39:17 +02:00
|
|
|
if config.verbose:
|
2017-02-13 16:12:56 +01:00
|
|
|
self.log.debug("Loading json file to db: %s (file: %s)" % (inner_path, file))
|
2015-07-12 20:36:46 +02:00
|
|
|
try:
|
2017-02-09 01:56:20 +01:00
|
|
|
self.updateDbFile(inner_path, file)
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.error("Json %s load error: %s" % (inner_path, Debug.formatException(err)))
|
|
|
|
self.closeDb()
|
|
|
|
|
|
|
|
# Load and parse json file
|
|
|
|
def loadJson(self, inner_path):
|
2019-03-16 00:10:49 +01:00
|
|
|
with self.open(inner_path, "r", encoding="utf8") as file:
|
2015-07-12 20:36:46 +02:00
|
|
|
return json.load(file)
|
|
|
|
|
2017-10-03 15:30:10 +02:00
|
|
|
def formatJson(self, data):
|
2015-09-20 00:27:54 +02:00
|
|
|
content = json.dumps(data, indent=1, sort_keys=True)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-10-02 14:23:05 +02:00
|
|
|
# Make it a little more compact by removing unnecessary white space
|
2015-07-12 20:36:46 +02:00
|
|
|
def compact_dict(match):
|
2016-10-02 14:23:05 +02:00
|
|
|
if "\n" in match.group(0):
|
|
|
|
return match.group(0).replace(match.group(1), match.group(1).strip())
|
2016-09-29 13:01:02 +02:00
|
|
|
else:
|
|
|
|
return match.group(0)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-10-02 14:23:05 +02:00
|
|
|
content = re.sub("\{(\n[^,\[\{]{10,100}?)\}[, ]{0,2}\n", compact_dict, content, flags=re.DOTALL)
|
Rev900, Sidebar filestats bar width round fix, Sidebar WebGL not supported error, Sidebar optimalizations, Trayicon gray shadow, Trim end of line whitespace from json files, Fix testweb testcase, Implement experimental postMessage nonce security, Return None when testing external ip, Window opener security check and message, Increase timeout for large files
2016-02-10 02:30:04 +01:00
|
|
|
|
2017-06-03 00:53:59 +02:00
|
|
|
def compact_list(match):
|
|
|
|
if "\n" in match.group(0):
|
|
|
|
stripped_lines = re.sub("\n[ ]*", "", match.group(1))
|
|
|
|
return match.group(0).replace(match.group(1), stripped_lines)
|
|
|
|
else:
|
|
|
|
return match.group(0)
|
|
|
|
|
|
|
|
content = re.sub("\[([^\[\{]{2,300}?)\][, ]{0,2}\n", compact_list, content, flags=re.DOTALL)
|
|
|
|
|
Rev900, Sidebar filestats bar width round fix, Sidebar WebGL not supported error, Sidebar optimalizations, Trayicon gray shadow, Trim end of line whitespace from json files, Fix testweb testcase, Implement experimental postMessage nonce security, Return None when testing external ip, Window opener security check and message, Increase timeout for large files
2016-02-10 02:30:04 +01:00
|
|
|
# Remove end of line whitespace
|
|
|
|
content = re.sub("(?m)[ ]+$", "", content)
|
2017-10-03 15:30:10 +02:00
|
|
|
return content
|
Rev900, Sidebar filestats bar width round fix, Sidebar WebGL not supported error, Sidebar optimalizations, Trayicon gray shadow, Trim end of line whitespace from json files, Fix testweb testcase, Implement experimental postMessage nonce security, Return None when testing external ip, Window opener security check and message, Increase timeout for large files
2016-02-10 02:30:04 +01:00
|
|
|
|
2017-10-03 15:30:10 +02:00
|
|
|
# Write formatted json file
|
|
|
|
def writeJson(self, inner_path, data):
|
2015-07-12 20:36:46 +02:00
|
|
|
# Write to disk
|
2019-03-16 00:10:49 +01:00
|
|
|
self.write(inner_path, self.formatJson(data).encode("utf8"))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Get file size
|
|
|
|
def getSize(self, inner_path):
|
|
|
|
path = self.getPath(inner_path)
|
2016-04-20 23:28:21 +02:00
|
|
|
try:
|
2015-07-12 20:36:46 +02:00
|
|
|
return os.path.getsize(path)
|
2016-04-20 23:28:21 +02:00
|
|
|
except:
|
2015-07-12 20:36:46 +02:00
|
|
|
return 0
|
|
|
|
|
|
|
|
# File exist
|
|
|
|
def isFile(self, inner_path):
|
|
|
|
return os.path.isfile(self.getPath(inner_path))
|
|
|
|
|
2015-10-11 02:22:53 +02:00
|
|
|
# File or directory exist
|
|
|
|
def isExists(self, inner_path):
|
|
|
|
return os.path.exists(self.getPath(inner_path))
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Dir exist
|
|
|
|
def isDir(self, inner_path):
|
|
|
|
return os.path.isdir(self.getPath(inner_path))
|
|
|
|
|
|
|
|
# Security check and return path of site's file
|
|
|
|
def getPath(self, inner_path):
|
|
|
|
inner_path = inner_path.replace("\\", "/") # Windows separator fix
|
2015-09-28 22:07:26 +02:00
|
|
|
if not inner_path:
|
|
|
|
return self.directory
|
|
|
|
|
2017-01-11 13:12:35 +01:00
|
|
|
if ".." in inner_path:
|
2019-03-15 21:06:59 +01:00
|
|
|
raise Exception("File not allowed: %s" % inner_path)
|
2016-04-06 13:48:13 +02:00
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
return "%s/%s" % (self.directory, inner_path)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-08-16 11:51:00 +02:00
|
|
|
# Get site dir relative path
|
|
|
|
def getInnerPath(self, path):
|
2015-09-28 22:07:26 +02:00
|
|
|
if path == self.directory:
|
|
|
|
inner_path = ""
|
|
|
|
else:
|
2017-06-15 13:30:36 +02:00
|
|
|
if path.startswith(self.directory):
|
2018-09-02 02:24:07 +02:00
|
|
|
inner_path = path[len(self.directory) + 1:]
|
2017-06-15 13:30:36 +02:00
|
|
|
else:
|
2019-03-15 21:06:59 +01:00
|
|
|
raise Exception("File not allowed: %s" % path)
|
2015-08-16 11:51:00 +02:00
|
|
|
return inner_path
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Verify all files sha512sum using content.json
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
def verifyFiles(self, quick_check=False, add_optional=False, add_changed=True):
|
2015-07-12 20:36:46 +02:00
|
|
|
bad_files = []
|
2018-03-29 02:49:06 +02:00
|
|
|
back = defaultdict(int)
|
|
|
|
back["bad_files"] = bad_files
|
2016-03-12 23:07:06 +01:00
|
|
|
i = 0
|
2018-03-29 02:49:06 +02:00
|
|
|
self.log.debug("Verifing files...")
|
2015-10-11 02:22:53 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if not self.site.content_manager.contents.get("content.json"): # No content.json, download it first
|
2016-09-05 13:55:51 +02:00
|
|
|
self.log.debug("VerifyFile content.json not exists")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.site.needFile("content.json", update=True) # Force update to fix corrupt file
|
|
|
|
self.site.content_manager.loadContent() # Reload content.json
|
2019-03-15 21:06:59 +01:00
|
|
|
for content_inner_path, content in list(self.site.content_manager.contents.items()):
|
2018-03-29 02:49:06 +02:00
|
|
|
back["num_content"] += 1
|
2016-03-12 23:07:06 +01:00
|
|
|
i += 1
|
2016-03-16 00:40:19 +01:00
|
|
|
if i % 50 == 0:
|
2019-03-16 00:10:49 +01:00
|
|
|
time.sleep(0.001) # Context switch to avoid gevent hangs
|
2015-07-12 20:36:46 +02:00
|
|
|
if not os.path.isfile(self.getPath(content_inner_path)): # Missing content.json file
|
2018-03-29 02:49:06 +02:00
|
|
|
back["num_content_missing"] += 1
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug("[MISSING] %s" % content_inner_path)
|
|
|
|
bad_files.append(content_inner_path)
|
2015-10-11 02:22:53 +02:00
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
for file_relative_path in list(content.get("files", {}).keys()):
|
2018-03-29 02:49:06 +02:00
|
|
|
back["num_file"] += 1
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
|
2015-07-12 20:36:46 +02:00
|
|
|
file_inner_path = file_inner_path.strip("/") # Strip leading /
|
|
|
|
file_path = self.getPath(file_inner_path)
|
|
|
|
if not os.path.isfile(file_path):
|
2018-03-29 02:49:06 +02:00
|
|
|
back["num_file_missing"] += 1
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug("[MISSING] %s" % file_inner_path)
|
|
|
|
bad_files.append(file_inner_path)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if quick_check:
|
|
|
|
ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"]
|
2017-06-19 15:57:02 +02:00
|
|
|
if not ok:
|
|
|
|
err = "Invalid size"
|
2015-07-12 20:36:46 +02:00
|
|
|
else:
|
2017-06-19 15:57:02 +02:00
|
|
|
try:
|
|
|
|
ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2017-06-19 15:57:02 +02:00
|
|
|
ok = False
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
if not ok:
|
2018-03-29 02:49:06 +02:00
|
|
|
back["num_file_invalid"] += 1
|
2017-06-19 15:57:02 +02:00
|
|
|
self.log.debug("[INVALID] %s: %s" % (file_inner_path, err))
|
2016-08-26 11:40:22 +02:00
|
|
|
if add_changed or content.get("cert_user_id"): # If updating own site only add changed user files
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
bad_files.append(file_inner_path)
|
2015-10-11 02:22:53 +02:00
|
|
|
|
|
|
|
# Optional files
|
|
|
|
optional_added = 0
|
|
|
|
optional_removed = 0
|
2019-03-15 21:06:59 +01:00
|
|
|
for file_relative_path in list(content.get("files_optional", {}).keys()):
|
2018-03-29 02:49:06 +02:00
|
|
|
back["num_optional"] += 1
|
2016-11-07 23:46:54 +01:00
|
|
|
file_node = content["files_optional"][file_relative_path]
|
2015-10-11 02:22:53 +02:00
|
|
|
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
|
|
|
|
file_inner_path = file_inner_path.strip("/") # Strip leading /
|
|
|
|
file_path = self.getPath(file_inner_path)
|
2018-03-29 03:19:26 +02:00
|
|
|
hash_id = self.site.content_manager.hashfield.getHashId(file_node["sha512"])
|
2015-10-11 02:22:53 +02:00
|
|
|
if not os.path.isfile(file_path):
|
2018-03-29 03:19:26 +02:00
|
|
|
if self.site.content_manager.isDownloaded(file_inner_path, hash_id):
|
|
|
|
back["num_optional_removed"] += 1
|
|
|
|
self.log.debug("[OPTIONAL REMOVED] %s" % file_inner_path)
|
|
|
|
self.site.content_manager.optionalRemoved(file_inner_path, hash_id, file_node["size"])
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
if add_optional:
|
|
|
|
bad_files.append(file_inner_path)
|
2015-10-11 02:22:53 +02:00
|
|
|
continue
|
|
|
|
|
|
|
|
if quick_check:
|
|
|
|
ok = os.path.getsize(file_path) == content["files_optional"][file_relative_path]["size"]
|
|
|
|
else:
|
2017-06-19 15:57:02 +02:00
|
|
|
try:
|
|
|
|
ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2017-06-19 15:57:02 +02:00
|
|
|
ok = False
|
2015-10-11 02:22:53 +02:00
|
|
|
|
|
|
|
if ok:
|
2018-03-29 03:19:26 +02:00
|
|
|
if not self.site.content_manager.isDownloaded(file_inner_path, hash_id):
|
|
|
|
back["num_optional_added"] += 1
|
|
|
|
self.site.content_manager.optionalDownloaded(file_inner_path, hash_id, file_node["size"])
|
2016-11-07 23:46:54 +01:00
|
|
|
optional_added += 1
|
2018-03-29 03:19:26 +02:00
|
|
|
self.log.debug("[OPTIONAL FOUND] %s" % file_inner_path)
|
2015-10-11 02:22:53 +02:00
|
|
|
else:
|
2018-03-29 03:19:26 +02:00
|
|
|
if self.site.content_manager.isDownloaded(file_inner_path, hash_id):
|
|
|
|
back["num_optional_removed"] += 1
|
|
|
|
self.site.content_manager.optionalRemoved(file_inner_path, hash_id, file_node["size"])
|
2016-11-07 23:46:54 +01:00
|
|
|
optional_removed += 1
|
|
|
|
bad_files.append(file_inner_path)
|
2015-10-11 02:22:53 +02:00
|
|
|
self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path)
|
|
|
|
|
2016-03-06 00:55:50 +01:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug(
|
2016-11-07 23:46:54 +01:00
|
|
|
"%s verified: %s, quick: %s, optionals: +%s -%s" %
|
|
|
|
(content_inner_path, len(content["files"]), quick_check, optional_added, optional_removed)
|
2016-03-06 00:55:50 +01:00
|
|
|
)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2018-03-29 03:19:26 +02:00
|
|
|
self.site.content_manager.contents.db.processDelayed()
|
2019-03-15 21:06:59 +01:00
|
|
|
time.sleep(0.001) # Context switch to avoid gevent hangs
|
2018-03-29 02:49:06 +02:00
|
|
|
return back
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Check and try to fix site files integrity
|
2016-09-04 17:59:29 +02:00
|
|
|
def updateBadFiles(self, quick_check=True):
|
2015-07-12 20:36:46 +02:00
|
|
|
s = time.time()
|
2018-03-29 02:49:06 +02:00
|
|
|
res = self.verifyFiles(
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
quick_check,
|
2016-11-07 23:46:54 +01:00
|
|
|
add_optional=self.site.isDownloadable(""),
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
add_changed=not self.site.settings.get("own") # Don't overwrite changed files if site owned
|
|
|
|
)
|
2018-03-29 02:49:06 +02:00
|
|
|
bad_files = res["bad_files"]
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
self.site.bad_files = {}
|
2015-07-12 20:36:46 +02:00
|
|
|
if bad_files:
|
|
|
|
for bad_file in bad_files:
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
self.site.bad_files[bad_file] = 1
|
2016-03-11 12:39:39 +01:00
|
|
|
self.log.debug("Checked files in %.2fs... Found bad files: %s, Quick:%s" % (time.time() - s, len(bad_files), quick_check))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Delete site's all file
|
|
|
|
def deleteFiles(self):
|
|
|
|
self.log.debug("Deleting files from content.json...")
|
|
|
|
files = [] # Get filenames
|
2019-03-15 21:06:59 +01:00
|
|
|
for content_inner_path in list(self.site.content_manager.contents.keys()):
|
2018-04-28 21:49:12 +02:00
|
|
|
content = self.site.content_manager.contents.get(content_inner_path, {})
|
2015-07-12 20:36:46 +02:00
|
|
|
files.append(content_inner_path)
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
# Add normal files
|
2019-03-15 21:06:59 +01:00
|
|
|
for file_relative_path in list(content.get("files", {}).keys()):
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
|
|
|
|
files.append(file_inner_path)
|
|
|
|
# Add optional files
|
2019-03-15 21:06:59 +01:00
|
|
|
for file_relative_path in list(content.get("files_optional", {}).keys()):
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to site dir
|
2015-07-12 20:36:46 +02:00
|
|
|
files.append(file_inner_path)
|
|
|
|
|
2016-08-10 12:31:54 +02:00
|
|
|
if self.isFile("dbschema.json"):
|
|
|
|
self.log.debug("Deleting db file...")
|
|
|
|
self.closeDb()
|
|
|
|
self.has_db = False
|
|
|
|
try:
|
|
|
|
schema = self.loadJson("dbschema.json")
|
|
|
|
db_path = self.getPath(schema["db_file"])
|
|
|
|
if os.path.isfile(db_path):
|
|
|
|
os.unlink(db_path)
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2016-08-10 12:31:54 +02:00
|
|
|
self.log.error("Db file delete error: %s" % err)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
for inner_path in files:
|
|
|
|
path = self.getPath(inner_path)
|
|
|
|
if os.path.isfile(path):
|
2016-11-07 23:47:09 +01:00
|
|
|
for retry in range(5):
|
|
|
|
try:
|
|
|
|
os.unlink(path)
|
|
|
|
break
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
|
|
|
self.log.error("Error removing %s: %s, try #%s" % (inner_path, err, retry))
|
2017-01-11 13:12:35 +01:00
|
|
|
time.sleep(float(retry) / 10)
|
2016-08-10 12:31:54 +02:00
|
|
|
self.onUpdated(inner_path, False)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
self.log.debug("Deleting empty dirs...")
|
|
|
|
for root, dirs, files in os.walk(self.directory, topdown=False):
|
|
|
|
for dir in dirs:
|
|
|
|
path = os.path.join(root, dir)
|
|
|
|
if os.path.isdir(path) and os.listdir(path) == []:
|
2018-02-08 18:03:55 +01:00
|
|
|
os.rmdir(path)
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug("Removing %s" % path)
|
|
|
|
if os.path.isdir(self.directory) and os.listdir(self.directory) == []:
|
2018-02-08 18:03:55 +01:00
|
|
|
os.rmdir(self.directory) # Remove sites directory if empty
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
if os.path.isdir(self.directory):
|
|
|
|
self.log.debug("Some unknown file remained in site data dir: %s..." % self.directory)
|
|
|
|
return False # Some files not deleted
|
|
|
|
else:
|
|
|
|
self.log.debug("Site data directory deleted: %s..." % self.directory)
|
2016-03-06 00:55:50 +01:00
|
|
|
return True # All clean
|