import os import json import logging import collections import time import hashlib from Debug import Debug from Plugin import PluginManager from Config import config from util import helper class ContentFilterStorage(object): def __init__(self, site_manager): self.log = logging.getLogger("ContentFilterStorage") self.file_path = "%s/filters.json" % config.data_dir self.site_manager = site_manager self.file_content = self.load() # Set default values for filters.json if not self.file_content: self.file_content = {} # Site blacklist renamed to site blocks if "site_blacklist" in self.file_content: self.file_content["siteblocks"] = self.file_content["site_blacklist"] del self.file_content["site_blacklist"] for key in ["mutes", "siteblocks", "includes"]: if key not in self.file_content: self.file_content[key] = {} self.include_filters = collections.defaultdict(set) # Merged list of mutes and blacklists from all include self.includeUpdateAll(update_site_dbs=False) def load(self): # Rename previously used mutes.json -> filters.json if os.path.isfile("%s/mutes.json" % config.data_dir): self.log.info("Renaming mutes.json to filters.json...") os.rename("%s/mutes.json" % config.data_dir, self.file_path) if os.path.isfile(self.file_path): try: return json.load(open(self.file_path)) except Exception as err: self.log.error("Error loading filters.json: %s" % err) return None else: return None def includeUpdateAll(self, update_site_dbs=True): s = time.time() new_include_filters = collections.defaultdict(set) # Load all include files data into a merged set for include_path in self.file_content["includes"]: address, inner_path = include_path.split("/", 1) try: content = self.site_manager.get(address).storage.loadJson(inner_path) except Exception as err: self.log.warning( "Error loading include %s: %s" % (include_path, Debug.formatException(err)) ) continue for key, val in content.items(): if type(val) is not dict: continue new_include_filters[key].update(val.keys()) mutes_added = new_include_filters["mutes"].difference(self.include_filters["mutes"]) mutes_removed = self.include_filters["mutes"].difference(new_include_filters["mutes"]) self.include_filters = new_include_filters if update_site_dbs: for auth_address in mutes_added: self.changeDbs(auth_address, "remove") for auth_address in mutes_removed: if not self.isMuted(auth_address): self.changeDbs(auth_address, "load") num_mutes = len(self.include_filters["mutes"]) num_siteblocks = len(self.include_filters["siteblocks"]) self.log.debug( "Loaded %s mutes, %s blocked sites from %s includes in %.3fs" % (num_mutes, num_siteblocks, len(self.file_content["includes"]), time.time() - s) ) def includeAdd(self, address, inner_path, description=None): self.file_content["includes"]["%s/%s" % (address, inner_path)] = { "date_added": time.time(), "address": address, "description": description, "inner_path": inner_path } self.includeUpdateAll() self.save() def includeRemove(self, address, inner_path): del self.file_content["includes"]["%s/%s" % (address, inner_path)] self.includeUpdateAll() self.save() def save(self): s = time.time() helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True).encode("utf8")) self.log.debug("Saved in %.3fs" % (time.time() - s)) def isMuted(self, auth_address): if auth_address in self.file_content["mutes"] or auth_address in self.include_filters["mutes"]: return True else: return False def getSiteAddressHashed(self, address): return "0x" + hashlib.sha256(address.encode("ascii")).hexdigest() def isSiteblocked(self, address): if address in self.file_content["siteblocks"] or address in self.include_filters["siteblocks"]: return True return False def getSiteblockDetails(self, address): details = self.file_content["siteblocks"].get(address) if not details: address_sha256 = self.getSiteAddressHashed(address) details = self.file_content["siteblocks"].get(address_sha256) if not details: includes = self.file_content.get("includes", {}).values() for include in includes: include_site = self.site_manager.get(include["address"]) if not include_site: continue content = include_site.storage.loadJson(include["inner_path"]) details = content.get("siteblocks", {}).get(address) if details: details["include"] = include break return details # Search and remove or readd files of an user def changeDbs(self, auth_address, action): self.log.debug("Mute action %s on user %s" % (action, auth_address)) res = list(self.site_manager.list().values())[0].content_manager.contents.db.execute( "SELECT * FROM content LEFT JOIN site USING (site_id) WHERE inner_path LIKE :inner_path", {"inner_path": "%%/%s/%%" % auth_address} ) for row in res: site = self.site_manager.sites.get(row["address"]) if not site: continue dir_inner_path = helper.getDirname(row["inner_path"]) for file_name in site.storage.walk(dir_inner_path): if action == "remove": site.storage.onUpdated(dir_inner_path + file_name, False) else: site.storage.onUpdated(dir_inner_path + file_name) site.onFileDone(dir_inner_path + file_name)