ZeroNet/src/Site/SiteManager.py

import json
import logging
import re
import os
import time
import atexit

import gevent

import util
from Plugin import PluginManager
from Content import ContentDb
from Config import config
from util import helper
from util import RateLimit
from util import Cached


@PluginManager.acceptPlugins
class SiteManager(object):
    def __init__(self):
        self.log = logging.getLogger("SiteManager")
        self.log.debug("SiteManager created.")
        self.sites = {}
        self.sites_changed = int(time.time())
        self.loaded = False
        gevent.spawn(self.saveTimer)
        atexit.register(lambda: self.save(recalculate_size=True))

    # Load all sites from data/sites.json
    @util.Noparallel()
    def load(self, cleanup=True, startup=False):
        from Debug import Debug
        self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup))
        self.loaded = False
        from .Site import Site
        address_found = []
        added = 0
        load_s = time.time()
        # Load new adresses
        try:
            json_path = "%s/sites.json" % config.data_dir
            data = json.load(open(json_path))
        except Exception as err:
            raise Exception("Unable to load %s: %s" % (json_path, err))

        sites_need = []

        for address, settings in data.items():
            if address not in self.sites:
                if os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
                    # Root content.json exists, try load site
                    s = time.time()
                    try:
                        site = Site(address, settings=settings)
                        site.content_manager.contents.get("content.json")
                    except Exception as err:
                        self.log.debug("Error loading site %s: %s" % (address, err))
                        continue
                    self.sites[address] = site
                    self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
                    added += 1
                elif startup:
                    # No site directory, start download
                    self.log.debug("Found new site in sites.json: %s" % address)
                    sites_need.append([address, settings])
                    added += 1

            address_found.append(address)

        # Remove deleted adresses
        if cleanup:
            for address in list(self.sites.keys()):
                if address not in address_found:
                    del(self.sites[address])
                    self.log.debug("Removed site: %s" % address)

            # Remove orpan sites from contentdb
            content_db = ContentDb.getContentDb()
            for row in content_db.execute("SELECT * FROM site").fetchall():
                address = row["address"]
                if address not in self.sites and address not in address_found:
                    self.log.info("Deleting orphan site from content.db: %s" % address)

                    try:
                        content_db.execute("DELETE FROM site WHERE ?", {"address": address})
                    except Exception as err:
                        self.log.error("Can't delete site %s from content_db: %s" % (address, err))

                    if address in content_db.site_ids:
                        del content_db.site_ids[address]
                    if address in content_db.sites:
                        del content_db.sites[address]

        self.loaded = True
        for address, settings in sites_need:
            gevent.spawn(self.need, address, settings=settings)
        if added:
            self.log.info("Added %s sites in %.3fs" % (added, time.time() - load_s))

    def saveDelayed(self):
        RateLimit.callAsync("Save sites.json", allowed_again=5, func=self.save)

    def save(self, recalculate_size=False):
        if not self.sites:
            self.log.debug("Save skipped: No sites found")
            return
        if not self.loaded:
            self.log.debug("Save skipped: Not loaded")
            return
        s = time.time()
        data = {}
        # Generate data file
        s = time.time()
        for address, site in list(self.list().items()):
            if recalculate_size:
                site.settings["size"], site.settings["size_optional"] = site.content_manager.getTotalSize()  # Update site size
            data[address] = site.settings
            data[address]["cache"] = site.getSettingsCache()
        time_generate = time.time() - s

        s = time.time()
        if data:
            helper.atomicWrite("%s/sites.json" % config.data_dir, helper.jsonDumps(data).encode("utf8"))
        else:
            self.log.debug("Save error: No data")
        time_write = time.time() - s

        # Remove cache from site settings
        for address, site in self.list().items():
            site.settings["cache"] = {}

        self.log.debug("Saved sites in %.2fs (generate: %.2fs, write: %.2fs)" % (time.time() - s, time_generate, time_write))

    def saveTimer(self):
        while 1:
            time.sleep(60 * 10)
            self.save(recalculate_size=True)

    # Checks if its a valid address
    def isAddress(self, address):
        return re.match("^[A-Za-z0-9]{26,35}$", address)

    def isDomain(self, address):
        return False

    @Cached(timeout=10)
    def isDomainCached(self, address):
        return self.isDomain(address)

    def resolveDomain(self, domain):
        return False

    @Cached(timeout=10)
    def resolveDomainCached(self, domain):
        return self.resolveDomain(domain)

    # Return: Site object or None if not found
    def get(self, address):
        if self.isDomainCached(address):
            address_resolved = self.resolveDomainCached(address)
            if address_resolved:
                address = address_resolved

        if not self.loaded:  # Not loaded yet
            self.log.debug("Loading site: %s)..." % address)
            self.load()
        site = self.sites.get(address)

        return site

    def add(self, address, all_file=True, settings=None, **kwargs):
        from .Site import Site
        self.sites_changed = int(time.time())
        # Try to find site with differect case
        for recover_address, recover_site in list(self.sites.items()):
            if recover_address.lower() == address.lower():
                return recover_site

        if not self.isAddress(address):
            return False  # Not address: %s % address
        self.log.debug("Added new site: %s" % address)
        config.loadTrackersFile()
        site = Site(address, settings=settings)
        self.sites[address] = site
        if not site.settings["serving"]:  # Maybe it was deleted before
            site.settings["serving"] = True
        site.saveSettings()
        if all_file:  # Also download user files on first sync
            site.download(check_size=True, blind_includes=True)
        return site

    # Return or create site and start download site files
    def need(self, address, *args, **kwargs):
        if self.isDomainCached(address):
            address_resolved = self.resolveDomainCached(address)
            if address_resolved:
                address = address_resolved

        site = self.get(address)
        if not site:  # Site not exist yet
            site = self.add(address, *args, **kwargs)
        return site

    def delete(self, address):
        self.sites_changed = int(time.time())
        self.log.debug("Deleted site: %s" % address)
        del(self.sites[address])
        # Delete from sites.json
        self.save()

    # Lazy load sites
    def list(self):
        if not self.loaded:  # Not loaded yet
            self.log.debug("Sites not loaded yet...")
            self.load(startup=True)
        return self.sites


site_manager = SiteManager()  # Singletone

if config.action == "main":  # Don't connect / add myself to peerlist
    peer_blacklist = [("127.0.0.1", config.fileserver_port), ("::1", config.fileserver_port)]
else:
    peer_blacklist = []