2015-07-12 20:36:46 +02:00
|
|
|
import json
|
|
|
|
import logging
|
|
|
|
import re
|
|
|
|
import os
|
Rev903, FeedQuery command only available for ADMIN sites, Show bad files in sidebar, Log unknown messages, Add and check inner_path and site address on sign/verify, Better peer cleanup limit, Log site load times, Testcase for address and inner_path verification, Re-sign testsite with new fields, Fix unnecessary loading screen display when browsing sub-folder with index.html, Fix safari notification width
2016-02-18 11:22:21 +01:00
|
|
|
import time
|
2016-09-04 17:51:20 +02:00
|
|
|
import atexit
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-04-20 23:28:58 +02:00
|
|
|
import gevent
|
|
|
|
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
from Plugin import PluginManager
|
2016-09-04 17:43:30 +02:00
|
|
|
from Content import ContentDb
|
2015-05-31 15:52:21 +02:00
|
|
|
from Config import config
|
2015-10-18 22:58:02 +02:00
|
|
|
from util import helper
|
2015-09-11 02:25:37 +02:00
|
|
|
|
2016-04-20 23:28:58 +02:00
|
|
|
|
version 0.2.8, Namecoin domains using internal resolver site, --disable_zeromq option to skip backward compatiblity layer and save some memory, connectionserver firstchar error fixes, missing unpacker crash fix, sitemanager class to allow extensions, add loaded plugin list to websocket api, faster content publishing, mark updating file as bad, remove coppersurfer tracker add eddie4, internal server error with error displaying, allow site domains in UiRequest, better progress bar, wait for siteinfo before before using localstorage, csslater hide only if opacity is 0
2015-03-30 23:44:29 +02:00
|
|
|
@PluginManager.acceptPlugins
|
|
|
|
class SiteManager(object):
|
2015-07-12 20:36:46 +02:00
|
|
|
def __init__(self):
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log = logging.getLogger("SiteManager")
|
2016-08-16 18:14:02 +02:00
|
|
|
self.log.debug("SiteManager created.")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.sites = None
|
2016-04-20 23:28:58 +02:00
|
|
|
gevent.spawn(self.saveTimer)
|
2016-09-04 17:51:20 +02:00
|
|
|
atexit.register(self.save)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Load all sites from data/sites.json
|
|
|
|
def load(self):
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.debug("Loading sites...")
|
2015-07-12 20:36:46 +02:00
|
|
|
from Site import Site
|
|
|
|
if not self.sites:
|
|
|
|
self.sites = {}
|
|
|
|
address_found = []
|
|
|
|
added = 0
|
|
|
|
# Load new adresses
|
2016-09-04 17:51:55 +02:00
|
|
|
for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems():
|
2015-07-12 20:36:46 +02:00
|
|
|
if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
|
Rev903, FeedQuery command only available for ADMIN sites, Show bad files in sidebar, Log unknown messages, Add and check inner_path and site address on sign/verify, Better peer cleanup limit, Log site load times, Testcase for address and inner_path verification, Re-sign testsite with new fields, Fix unnecessary loading screen display when browsing sub-folder with index.html, Fix safari notification width
2016-02-18 11:22:21 +01:00
|
|
|
s = time.time()
|
2016-09-04 17:51:55 +02:00
|
|
|
self.sites[address] = Site(address, settings=settings)
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
|
2015-07-12 20:36:46 +02:00
|
|
|
added += 1
|
|
|
|
address_found.append(address)
|
|
|
|
|
|
|
|
# Remove deleted adresses
|
|
|
|
for address in self.sites.keys():
|
|
|
|
if address not in address_found:
|
|
|
|
del(self.sites[address])
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.debug("Removed site: %s" % address)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-09-04 17:52:14 +02:00
|
|
|
# Remove orpan sites from contentdb
|
2016-09-05 13:57:12 +02:00
|
|
|
for row in ContentDb.getContentDb().execute("SELECT * FROM site"):
|
|
|
|
if row["address"] not in self.sites:
|
|
|
|
self.log.info("Deleting orphan site from content.db: %s" % row["address"])
|
|
|
|
ContentDb.getContentDb().deleteSite(row["address"])
|
2016-09-04 17:52:14 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if added:
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.debug("SiteManager added %s sites" % added)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-04-20 23:28:58 +02:00
|
|
|
def save(self):
|
|
|
|
if not self.sites:
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.error("Save error: No sites found")
|
2016-04-20 23:28:58 +02:00
|
|
|
s = time.time()
|
2016-08-10 12:33:34 +02:00
|
|
|
data = {}
|
2016-09-04 17:43:30 +02:00
|
|
|
# Generate data file
|
2016-04-20 23:28:58 +02:00
|
|
|
for address, site in self.list().iteritems():
|
2016-09-04 17:43:30 +02:00
|
|
|
site.settings["size"] = site.content_manager.getTotalSize() # Update site size
|
2016-04-20 23:28:58 +02:00
|
|
|
data[address] = site.settings
|
2016-09-04 17:43:30 +02:00
|
|
|
data[address]["cache"] = {}
|
|
|
|
data[address]["cache"]["bad_files"] = site.bad_files
|
|
|
|
data[address]["cache"]["hashfield"] = site.content_manager.hashfield.tostring().encode("base64")
|
|
|
|
|
2016-04-20 23:28:58 +02:00
|
|
|
helper.atomicWrite("%s/sites.json" % config.data_dir, json.dumps(data, indent=2, sort_keys=True))
|
2016-09-04 17:43:30 +02:00
|
|
|
# Remove cache from site settings
|
|
|
|
for address, site in self.list().iteritems():
|
|
|
|
site.settings["cache"] = {}
|
|
|
|
|
2016-08-10 12:33:34 +02:00
|
|
|
self.log.debug("Saved sites in %.2fs" % (time.time() - s))
|
2016-04-20 23:28:58 +02:00
|
|
|
|
|
|
|
def saveTimer(self):
|
|
|
|
while 1:
|
|
|
|
time.sleep(60 * 10)
|
|
|
|
self.save()
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Checks if its a valid address
|
|
|
|
def isAddress(self, address):
|
|
|
|
return re.match("^[A-Za-z0-9]{26,35}$", address)
|
|
|
|
|
|
|
|
# Return: Site object or None if not found
|
|
|
|
def get(self, address):
|
|
|
|
if self.sites is None: # Not loaded yet
|
2016-08-10 12:34:28 +02:00
|
|
|
self.log.debug("Getting new site: %s)..." % address)
|
2015-07-12 20:36:46 +02:00
|
|
|
self.load()
|
|
|
|
return self.sites.get(address)
|
|
|
|
|
|
|
|
# Return or create site and start download site files
|
|
|
|
def need(self, address, all_file=True):
|
|
|
|
from Site import Site
|
|
|
|
site = self.get(address)
|
|
|
|
if not site: # Site not exist yet
|
2016-03-09 15:30:04 +01:00
|
|
|
# Try to find site with differect case
|
|
|
|
for recover_address, recover_site in self.sites.items():
|
|
|
|
if recover_address.lower() == address.lower():
|
|
|
|
return recover_site
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if not self.isAddress(address):
|
|
|
|
return False # Not address: %s % address
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.debug("Added new site: %s" % address)
|
2015-07-12 20:36:46 +02:00
|
|
|
site = Site(address)
|
|
|
|
self.sites[address] = site
|
|
|
|
if not site.settings["serving"]: # Maybe it was deleted before
|
|
|
|
site.settings["serving"] = True
|
2016-08-10 12:34:28 +02:00
|
|
|
site.saveSettings()
|
2015-08-06 00:51:25 +02:00
|
|
|
if all_file: # Also download user files on first sync
|
|
|
|
site.download(blind_includes=True)
|
|
|
|
else:
|
|
|
|
if all_file:
|
|
|
|
site.download()
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
return site
|
|
|
|
|
|
|
|
def delete(self, address):
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.debug("SiteManager deleted site: %s" % address)
|
2015-07-12 20:36:46 +02:00
|
|
|
del(self.sites[address])
|
2015-09-11 02:25:37 +02:00
|
|
|
# Delete from sites.json
|
2016-08-10 12:34:28 +02:00
|
|
|
self.save()
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Lazy load sites
|
|
|
|
def list(self):
|
|
|
|
if self.sites is None: # Not loaded yet
|
2016-08-10 12:32:28 +02:00
|
|
|
self.log.debug("Sites not loaded yet...")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.load()
|
|
|
|
return self.sites
|
|
|
|
|
|
|
|
|
|
|
|
site_manager = SiteManager() # Singletone
|
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
peer_blacklist = [("127.0.0.1", config.fileserver_port)] # Dont add this peers
|