223 lines
7.8 KiB
Python
223 lines
7.8 KiB
Python
import json
|
|
import logging
|
|
import re
|
|
import os
|
|
import time
|
|
import atexit
|
|
|
|
import gevent
|
|
|
|
import util
|
|
from Plugin import PluginManager
|
|
from Content import ContentDb
|
|
from Config import config
|
|
from util import helper
|
|
from util import RateLimit
|
|
from util import Cached
|
|
|
|
|
|
@PluginManager.acceptPlugins
|
|
class SiteManager(object):
|
|
def __init__(self):
|
|
self.log = logging.getLogger("SiteManager")
|
|
self.log.debug("SiteManager created.")
|
|
self.sites = {}
|
|
self.sites_changed = int(time.time())
|
|
self.loaded = False
|
|
gevent.spawn(self.saveTimer)
|
|
atexit.register(lambda: self.save(recalculate_size=True))
|
|
|
|
# Load all sites from data/sites.json
|
|
@util.Noparallel()
|
|
def load(self, cleanup=True, startup=False):
|
|
from Debug import Debug
|
|
self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup))
|
|
self.loaded = False
|
|
from .Site import Site
|
|
address_found = []
|
|
added = 0
|
|
load_s = time.time()
|
|
# Load new adresses
|
|
try:
|
|
json_path = "%s/sites.json" % config.data_dir
|
|
data = json.load(open(json_path))
|
|
except Exception as err:
|
|
raise Exception("Unable to load %s: %s" % (json_path, err))
|
|
|
|
for address, settings in data.items():
|
|
if address not in self.sites:
|
|
if os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
|
|
# Root content.json exists, try load site
|
|
s = time.time()
|
|
try:
|
|
site = Site(address, settings=settings)
|
|
site.content_manager.contents.get("content.json")
|
|
except Exception as err:
|
|
self.log.debug("Error loading site %s: %s" % (address, err))
|
|
continue
|
|
self.sites[address] = site
|
|
self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
|
|
added += 1
|
|
elif startup:
|
|
# No site directory, start download
|
|
self.log.debug("Found new site in sites.json: %s" % address)
|
|
gevent.spawn(self.need, address, settings=settings)
|
|
added += 1
|
|
|
|
address_found.append(address)
|
|
|
|
# Remove deleted adresses
|
|
if cleanup:
|
|
for address in list(self.sites.keys()):
|
|
if address not in address_found:
|
|
del(self.sites[address])
|
|
self.log.debug("Removed site: %s" % address)
|
|
|
|
# Remove orpan sites from contentdb
|
|
content_db = ContentDb.getContentDb()
|
|
for row in content_db.execute("SELECT * FROM site").fetchall():
|
|
address = row["address"]
|
|
if address not in self.sites:
|
|
self.log.info("Deleting orphan site from content.db: %s" % address)
|
|
|
|
try:
|
|
content_db.execute("DELETE FROM site WHERE ?", {"address": address})
|
|
except Exception as err:
|
|
self.log.error("Can't delete site %s from content_db: %s" % (address, err))
|
|
|
|
if address in content_db.site_ids:
|
|
del content_db.site_ids[address]
|
|
if address in content_db.sites:
|
|
del content_db.sites[address]
|
|
|
|
if added:
|
|
self.log.info("Added %s sites in %.3fs" % (added, time.time() - load_s))
|
|
self.loaded = True
|
|
|
|
def saveDelayed(self):
|
|
RateLimit.callAsync("Save sites.json", allowed_again=5, func=self.save)
|
|
|
|
def save(self, recalculate_size=False):
|
|
if not self.sites:
|
|
self.log.debug("Save skipped: No sites found")
|
|
return
|
|
if not self.loaded:
|
|
self.log.debug("Save skipped: Not loaded")
|
|
return
|
|
s = time.time()
|
|
data = {}
|
|
# Generate data file
|
|
s = time.time()
|
|
for address, site in list(self.list().items()):
|
|
if recalculate_size:
|
|
site.settings["size"], site.settings["size_optional"] = site.content_manager.getTotalSize() # Update site size
|
|
data[address] = site.settings
|
|
data[address]["cache"] = site.getSettingsCache()
|
|
time_generate = time.time() - s
|
|
|
|
s = time.time()
|
|
if data:
|
|
helper.atomicWrite("%s/sites.json" % config.data_dir, helper.jsonDumps(data).encode("utf8"))
|
|
else:
|
|
self.log.debug("Save error: No data")
|
|
time_write = time.time() - s
|
|
|
|
# Remove cache from site settings
|
|
for address, site in self.list().items():
|
|
site.settings["cache"] = {}
|
|
|
|
self.log.debug("Saved sites in %.2fs (generate: %.2fs, write: %.2fs)" % (time.time() - s, time_generate, time_write))
|
|
|
|
def saveTimer(self):
|
|
while 1:
|
|
time.sleep(60 * 10)
|
|
self.save(recalculate_size=True)
|
|
|
|
# Checks if its a valid address
|
|
def isAddress(self, address):
|
|
return re.match("^[A-Za-z0-9]{26,35}$", address)
|
|
|
|
def isDomain(self, address):
|
|
return False
|
|
|
|
@Cached(timeout=10)
|
|
def isDomainCached(self, address):
|
|
return self.isDomain(address)
|
|
|
|
def resolveDomain(self, domain):
|
|
return False
|
|
|
|
@Cached(timeout=10)
|
|
def resolveDomainCached(self, domain):
|
|
return self.resolveDomain(domain)
|
|
|
|
# Return: Site object or None if not found
|
|
def get(self, address):
|
|
if self.isDomainCached(address):
|
|
address_resolved = self.resolveDomainCached(address)
|
|
if address_resolved:
|
|
address = address_resolved
|
|
|
|
if not self.loaded: # Not loaded yet
|
|
self.log.debug("Loading site: %s)..." % address)
|
|
self.load()
|
|
site = self.sites.get(address)
|
|
|
|
return site
|
|
|
|
def add(self, address, all_file=False, settings=None, **kwargs):
|
|
from .Site import Site
|
|
self.sites_changed = int(time.time())
|
|
# Try to find site with differect case
|
|
for recover_address, recover_site in list(self.sites.items()):
|
|
if recover_address.lower() == address.lower():
|
|
return recover_site
|
|
|
|
if not self.isAddress(address):
|
|
return False # Not address: %s % address
|
|
self.log.debug("Added new site: %s" % address)
|
|
config.loadTrackersFile()
|
|
site = Site(address, settings=settings)
|
|
self.sites[address] = site
|
|
if not site.settings["serving"]: # Maybe it was deleted before
|
|
site.settings["serving"] = True
|
|
site.saveSettings()
|
|
if all_file: # Also download user files on first sync
|
|
site.download(check_size=True, blind_includes=True)
|
|
return site
|
|
|
|
# Return or create site and start download site files
|
|
def need(self, address, *args, **kwargs):
|
|
if self.isDomainCached(address):
|
|
address_resolved = self.resolveDomainCached(address)
|
|
if address_resolved:
|
|
address = address_resolved
|
|
|
|
site = self.get(address)
|
|
if not site: # Site not exist yet
|
|
site = self.add(address, *args, **kwargs)
|
|
return site
|
|
|
|
def delete(self, address):
|
|
self.sites_changed = int(time.time())
|
|
self.log.debug("Deleted site: %s" % address)
|
|
del(self.sites[address])
|
|
# Delete from sites.json
|
|
self.save()
|
|
|
|
# Lazy load sites
|
|
def list(self):
|
|
if not self.loaded: # Not loaded yet
|
|
self.log.debug("Sites not loaded yet...")
|
|
self.load(startup=True)
|
|
return self.sites
|
|
|
|
|
|
site_manager = SiteManager() # Singletone
|
|
|
|
if config.action == "main": # Don't connect / add myself to peerlist
|
|
peer_blacklist = [("127.0.0.1", config.fileserver_port), ("::1", config.fileserver_port)]
|
|
else:
|
|
peer_blacklist = []
|
|
|