New ContentFilter plugin for shared site and user blocklist

This commit is contained in:
shortcutme 2018-06-25 14:35:26 +02:00
parent a59fb4fd1e
commit c493f732f9
No known key found for this signature in database
GPG Key ID: 5B63BAE6CB9613AE
12 changed files with 660 additions and 0 deletions

View File

@ -0,0 +1,202 @@
import time
import re
from Plugin import PluginManager
from Translate import Translate
from Config import config
from ContentFilterStorage import ContentFilterStorage
if "_" not in locals():
_ = Translate("plugins/ContentFilter/languages/")
@PluginManager.registerTo("SiteManager")
class SiteManagerPlugin(object):
def load(self, *args, **kwargs):
global filter_storage
super(SiteManagerPlugin, self).load(*args, **kwargs)
filter_storage = ContentFilterStorage(site_manager=self)
@PluginManager.registerTo("UiWebsocket")
class UiWebsocketPlugin(object):
# Mute
def cbMuteAdd(self, to, auth_address, cert_user_id, reason):
filter_storage.file_content["mutes"][auth_address] = {
"cert_user_id": cert_user_id, "reason": reason, "source": self.site.address, "date_added": time.time()
}
filter_storage.save()
filter_storage.changeDbs(auth_address, "remove")
self.response(to, "ok")
def actionMuteAdd(self, to, auth_address, cert_user_id, reason):
if "ADMIN" in self.getPermissions(to):
self.cbMuteAdd(to, auth_address, cert_user_id, reason)
else:
self.cmd(
"confirm",
[_["Hide all content from <b>%s</b>?"] % cert_user_id, _["Mute"]],
lambda (res): self.cbMuteAdd(to, auth_address, cert_user_id, reason)
)
def cbMuteRemove(self, to, auth_address):
del filter_storage.file_content["mutes"][auth_address]
filter_storage.save()
filter_storage.changeDbs(auth_address, "load")
self.response(to, "ok")
def actionMuteRemove(self, to, auth_address):
if "ADMIN" in self.getPermissions(to):
self.cbMuteRemove(to, auth_address)
else:
self.cmd(
"confirm",
[_["Unmute <b>%s</b>?"] % filter_storage.file_content["mutes"][auth_address]["cert_user_id"], _["Unmute"]],
lambda (res): self.cbMuteRemove(to, auth_address)
)
def actionMuteList(self, to):
if "ADMIN" in self.getPermissions(to):
self.response(to, filter_storage.file_content["mutes"])
else:
return self.response(to, {"error": "Forbidden: Only ADMIN sites can list mutes"})
# Siteblock
def actionSiteblockAdd(self, to, site_address, reason=None):
if "ADMIN" not in self.getPermissions(to):
return self.response(to, {"error": "Forbidden: Only ADMIN sites can add to blocklist"})
filter_storage.file_content["siteblocks"][site_address] = {"date_added": time.time(), "reason": reason}
filter_storage.save()
self.response(to, "ok")
def actionSiteblockRemove(self, to, site_address):
if "ADMIN" not in self.getPermissions(to):
return self.response(to, {"error": "Forbidden: Only ADMIN sites can remove from blocklist"})
del filter_storage.file_content["siteblocks"][site_address]
filter_storage.save()
self.response(to, "ok")
def actionSiteblockList(self, to):
if "ADMIN" in self.getPermissions(to):
self.response(to, filter_storage.file_content["siteblocks"])
else:
return self.response(to, {"error": "Forbidden: Only ADMIN sites can list blocklists"})
# Include
def actionFilterIncludeAdd(self, to, inner_path, description=None, address=None):
if address:
if "ADMIN" not in self.getPermissions(to):
return self.response(to, {"error": "Forbidden: Only ADMIN sites can manage different site include"})
site = self.server.sites[address]
else:
address = self.site.address
site = self.site
if "ADMIN" in self.getPermissions(to):
self.cbFilterIncludeAdd(to, True, address, inner_path, description)
else:
content = site.storage.loadJson(inner_path)
title = _["New shared global content filter: <b>%s</b> (%s sites, %s users)"] % (
inner_path, len(content.get("siteblocks", {})), len(content.get("mutes", {}))
)
self.cmd(
"confirm",
[title, "Add"],
lambda (res): self.cbFilterIncludeAdd(to, res, address, inner_path, description)
)
def cbFilterIncludeAdd(self, to, res, address, inner_path, description):
if not res:
self.response(to, res)
return False
filter_storage.includeAdd(address, inner_path, description)
self.response(to, "ok")
def actionFilterIncludeRemove(self, to, inner_path, address=None):
if address:
if "ADMIN" not in self.getPermissions(to):
return self.response(to, {"error": "Forbidden: Only ADMIN sites can manage different site include"})
else:
address = self.site.address
key = "%s/%s" % (address, inner_path)
if key not in filter_storage.file_content["includes"]:
self.response(to, {"error": "Include not found"})
filter_storage.includeRemove(address, inner_path)
self.response(to, "ok")
def actionFilterIncludeList(self, to, all_sites=False, filters=False):
if all_sites and "ADMIN" not in self.getPermissions(to):
return self.response(to, {"error": "Forbidden: Only ADMIN sites can list all sites includes"})
back = []
includes = filter_storage.file_content.get("includes", {}).values()
for include in includes:
if not all_sites and include["address"] != self.site.address:
continue
if filters:
content = filter_storage.site_manager.get(include["address"]).storage.loadJson(include["inner_path"])
include["mutes"] = content.get("mutes", {})
include["siteblocks"] = content.get("siteblocks", {})
back.append(include)
self.response(to, back)
@PluginManager.registerTo("SiteStorage")
class SiteStoragePlugin(object):
def updateDbFile(self, inner_path, file=None, cur=None):
if file is not False: # File deletion always allowed
# Find for bitcoin addresses in file path
matches = re.findall("/(1[A-Za-z0-9]{26,35})/", inner_path)
# Check if any of the adresses are in the mute list
for auth_address in matches:
if filter_storage.isMuted(auth_address):
self.log.debug("Mute match: %s, ignoring %s" % (auth_address, inner_path))
return False
return super(SiteStoragePlugin, self).updateDbFile(inner_path, file=file, cur=cur)
def onUpdated(self, inner_path, file=None):
file_path = "%s/%s" % (self.site.address, inner_path)
if file_path in filter_storage.file_content["includes"]:
self.log.debug("Filter file updated: %s" % inner_path)
filter_storage.includeUpdateAll()
return super(SiteStoragePlugin, self).onUpdated(inner_path, file=file)
@PluginManager.registerTo("UiRequest")
class UiRequestPlugin(object):
def actionWrapper(self, path, extra_headers=None):
match = re.match("/(?P<address>[A-Za-z0-9\._-]+)(?P<inner_path>/.*|$)", path)
if not match:
return False
address = match.group("address")
if self.server.site_manager.get(address): # Site already exists
return super(UiRequestPlugin, self).actionWrapper(path, extra_headers)
if self.server.site_manager.isDomain(address):
address = self.server.site_manager.resolveDomain(address)
if filter_storage.isSiteblocked(address):
site = self.server.site_manager.get(config.homepage)
if not extra_headers:
extra_headers = {}
self.sendHeader(extra_headers=extra_headers)
return iter([super(UiRequestPlugin, self).renderWrapper(
site, path, "uimedia/plugins/contentfilter/blocklisted.html?address=" + address,
"Blacklisted site", extra_headers, show_loadingscreen=False
)])
else:
return super(UiRequestPlugin, self).actionWrapper(path, extra_headers)
def actionUiMedia(self, path, *args, **kwargs):
if path.startswith("/uimedia/plugins/contentfilter/"):
file_path = path.replace("/uimedia/plugins/contentfilter/", "plugins/ContentFilter/media/")
return self.actionFile(file_path)
else:
return super(UiRequestPlugin, self).actionUiMedia(path)

View File

@ -0,0 +1,140 @@
import os
import json
import logging
import collections
import time
from Debug import Debug
from Plugin import PluginManager
from Config import config
from util import helper
class ContentFilterStorage(object):
def __init__(self, site_manager):
self.log = logging.getLogger("ContentFilterStorage")
self.file_path = "%s/filters.json" % config.data_dir
self.site_manager = site_manager
self.file_content = self.load()
# Set default values for filters.json
if not self.file_content:
self.file_content = {}
# Site blacklist renamed to site blocks
if "site_blacklist" in self.file_content:
self.file_content["siteblocks"] = self.file_content["site_blacklist"]
del self.file_content["site_blacklist"]
for key in ["mutes", "siteblocks", "includes"]:
if key not in self.file_content:
self.file_content[key] = {}
self.include_filters = collections.defaultdict(set) # Merged list of mutes and blacklists from all include
self.includeUpdateAll(update_site_dbs=False)
def load(self):
# Rename previously used mutes.json -> filters.json
if os.path.isfile("%s/mutes.json" % config.data_dir):
self.log.info("Renaming mutes.json to filters.json...")
os.rename("%s/mutes.json" % config.data_dir, self.file_path)
if os.path.isfile(self.file_path):
try:
return json.load(open(self.file_path))
except Exception as err:
self.log.error("Error loading filters.json: %s" % err)
return None
else:
return None
def includeUpdateAll(self, update_site_dbs=True):
s = time.time()
new_include_filters = collections.defaultdict(set)
# Load all include files data into a merged set
for include_path in self.file_content["includes"]:
address, inner_path = include_path.split("/", 1)
try:
content = self.site_manager.get(address).storage.loadJson(inner_path)
except Exception as err:
self.log.warning(
"Error loading include %s: %s" %
(include_path, Debug.formatException(err))
)
continue
for key, val in content.iteritems():
if type(val) is not dict:
continue
new_include_filters[key].update(val.keys())
mutes_added = new_include_filters["mutes"].difference(self.include_filters["mutes"])
mutes_removed = self.include_filters["mutes"].difference(new_include_filters["mutes"])
self.include_filters = new_include_filters
if update_site_dbs:
for auth_address in mutes_added:
self.changeDbs(auth_address, "remove")
for auth_address in mutes_removed:
if not self.isMuted(auth_address):
self.changeDbs(auth_address, "load")
num_mutes = len(self.include_filters["mutes"])
num_siteblocks = len(self.include_filters["siteblocks"])
self.log.debug(
"Loaded %s mutes, %s blocked sites from %s includes in %.3fs" %
(num_mutes, num_siteblocks, len(self.file_content["includes"]), time.time() - s)
)
def includeAdd(self, address, inner_path, description=None):
self.file_content["includes"]["%s/%s" % (address, inner_path)] = {
"date_added": time.time(),
"address": address,
"description": description,
"inner_path": inner_path
}
self.includeUpdateAll()
self.save()
def includeRemove(self, address, inner_path):
del self.file_content["includes"]["%s/%s" % (address, inner_path)]
self.includeUpdateAll()
self.save()
def save(self):
s = time.time()
helper.atomicWrite(self.file_path, json.dumps(self.file_content, indent=2, sort_keys=True))
self.log.debug("Saved in %.3fs" % (time.time() - s))
def isMuted(self, auth_address):
if auth_address in self.file_content["mutes"] or auth_address in self.include_filters["mutes"]:
return True
else:
return False
def isSiteblocked(self, address):
if address in self.file_content["siteblocks"] or address in self.include_filters["siteblocks"]:
return True
else:
return False
# Search and remove or readd files of an user
def changeDbs(self, auth_address, action):
self.log.debug("Mute action %s on user %s" % (action, auth_address))
res = self.site_manager.list().values()[0].content_manager.contents.db.execute(
"SELECT * FROM content LEFT JOIN site USING (site_id) WHERE inner_path LIKE :inner_path",
{"inner_path": "%%/%s/%%" % auth_address}
)
for row in res:
site = self.site_manager.sites.get(row["address"])
if not site:
continue
dir_inner_path = helper.getDirname(row["inner_path"])
for file_name in site.storage.walk(dir_inner_path):
if action == "remove":
site.storage.onUpdated(dir_inner_path + file_name, False)
else:
site.storage.onUpdated(dir_inner_path + file_name)
site.onFileDone(dir_inner_path + file_name)

View File

@ -0,0 +1,82 @@
import pytest
from ContentFilter import ContentFilterPlugin
from Site import SiteManager
@pytest.fixture
def filter_storage():
ContentFilterPlugin.filter_storage = ContentFilterPlugin.ContentFilterStorage(SiteManager.site_manager)
return ContentFilterPlugin.filter_storage
@pytest.mark.usefixtures("resetSettings")
@pytest.mark.usefixtures("resetTempSettings")
class TestContentFilter:
def createInclude(self, site):
site.storage.writeJson("filters.json", {
"mutes": {"1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C": {}},
"siteblocks": {site.address: {}}
})
def testIncludeLoad(self, site, filter_storage):
self.createInclude(site)
filter_storage.file_content["includes"]["%s/%s" % (site.address, "filters.json")] = {
"date_added": 1528295893,
}
assert not filter_storage.include_filters["mutes"]
assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
assert not filter_storage.isSiteblocked(site.address)
filter_storage.includeUpdateAll(update_site_dbs=False)
assert len(filter_storage.include_filters["mutes"]) == 1
assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
assert filter_storage.isSiteblocked(site.address)
def testIncludeAdd(self, site, filter_storage):
self.createInclude(site)
query_num_json = "SELECT COUNT(*) AS num FROM json WHERE directory = 'users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C'"
assert not filter_storage.isSiteblocked(site.address)
assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
assert site.storage.query(query_num_json).fetchone()["num"] == 2
# Add include
filter_storage.includeAdd(site.address, "filters.json")
assert filter_storage.isSiteblocked(site.address)
assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
assert site.storage.query(query_num_json).fetchone()["num"] == 0
# Remove include
filter_storage.includeRemove(site.address, "filters.json")
assert not filter_storage.isSiteblocked(site.address)
assert not filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
assert site.storage.query(query_num_json).fetchone()["num"] == 2
def testIncludeChange(self, site, filter_storage):
self.createInclude(site)
filter_storage.includeAdd(site.address, "filters.json")
assert filter_storage.isSiteblocked(site.address)
assert filter_storage.isMuted("1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C")
# Add new blocked site
assert not filter_storage.isSiteblocked("1Hello")
filter_content = site.storage.loadJson("filters.json")
filter_content["siteblocks"]["1Hello"] = {}
site.storage.writeJson("filters.json", filter_content)
assert filter_storage.isSiteblocked("1Hello")
# Add new muted user
query_num_json = "SELECT COUNT(*) AS num FROM json WHERE directory = 'users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q'"
assert not filter_storage.isMuted("1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q")
assert site.storage.query(query_num_json).fetchone()["num"] == 2
filter_content["mutes"]["1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q"] = {}
site.storage.writeJson("filters.json", filter_content)
assert filter_storage.isMuted("1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q")
assert site.storage.query(query_num_json).fetchone()["num"] == 0

View File

@ -0,0 +1 @@
from src.Test.conftest import *

View File

@ -0,0 +1,5 @@
[pytest]
python_files = Test*.py
addopts = -rsxX -v --durations=6
markers =
webtest: mark a test as a webtest.

View File

@ -0,0 +1 @@
import ContentFilterPlugin

View File

@ -0,0 +1,6 @@
{
"Hide all content from <b>%s</b>?": "<b>%s</b> tartalmaniak elrejtése?",
"Mute": "Elnémítás",
"Unmute <b>%s</b>?": "<b>%s</b> tartalmaniak megjelenítése?",
"Unmute": "Némítás visszavonása"
}

View File

@ -0,0 +1,6 @@
{
"Hide all content from <b>%s</b>?": "<b>%s</b> Vuoi nascondere i contenuti di questo utente ?",
"Mute": "Attiva Silenzia",
"Unmute <b>%s</b>?": "<b>%s</b> Vuoi mostrare i contenuti di questo utente ?",
"Unmute": "Disattiva Silenzia"
}

View File

@ -0,0 +1,6 @@
{
"Hide all content from <b>%s</b>?": "屏蔽 <b>%s</b> 的所有內容?",
"Mute": "屏蔽",
"Unmute <b>%s</b>?": "對 <b>%s</b> 解除屏蔽?",
"Unmute": "解除屏蔽"
}

View File

@ -0,0 +1,6 @@
{
"Hide all content from <b>%s</b>?": "屏蔽 <b>%s</b> 的所有内容?",
"Mute": "屏蔽",
"Unmute <b>%s</b>?": "对 <b>%s</b> 解除屏蔽?",
"Unmute": "解除屏蔽"
}

View File

@ -0,0 +1,86 @@
<html>
<body>
<style>
.content { line-height: 24px; font-family: monospace; font-size: 14px; color: #636363; text-transform: uppercase; top: 38%; position: relative; text-align: center; perspective: 1000px }
.content h1, .content h2 { font-weight: normal; letter-spacing: 1px; }
.content h2 { font-size: 15px; }
.content #details {
text-align: left; display: inline-block; width: 350px; background-color: white; padding: 17px 27px; border-radius: 0px;
box-shadow: 0px 2px 7px -1px #d8d8d8; text-transform: none; margin: 15px; transform: scale(0) rotateX(90deg); transition: all 0.6s cubic-bezier(0.785, 0.135, 0.15, 0.86);
}
.content #details #added { font-size: 12px; text-align: right; color: #a9a9a9; }
#button { transition: all 1s cubic-bezier(0.075, 0.82, 0.165, 1); opacity: 0; transform: translateY(50px); transition-delay: 0.5s }
.button {
padding: 8px 20px; background-color: #FFF85F; border-bottom: 2px solid #CDBD1E; border-radius: 2px;
text-decoration: none; transition: all 0.5s; background-position: left center; display: inline-block; margin-top: 10px; color: black;
}
.button:hover { background-color: #FFF400; border-bottom: 2px solid #4D4D4C; transition: none; }
.button:active { position: relative; top: 1px; }
.button:focus { outline: none; }
</style>
<div class="content">
<h1>Site blocked</h1>
<h2>This site is on your blocklist:</h2>
<div id="details">
<div id="reason">Too much image</div>
<div id="added">on 2015-01-25 12:32:11</div>
</div>
<div><a href="#Visit+Site" class="button button-submit" id="button">Remove from blocklist</a></div>
</div>
<script type="text/javascript" src="js/ZeroFrame.js"></script>
<script>
class Page extends ZeroFrame {
onOpenWebsocket () {
this.cmd("wrapperSetTitle", "Visiting a blocked site - ZeroNet")
this.cmd("siteInfo", {}, (site_info) => {
this.site_info = site_info
})
var address = document.location.search.match(/address=(.*?)[&\?]/)[1]
this.updateSiteblockDetails(address)
}
async updateSiteblockDetails(address) {
var blocks = await this.cmdp("siteblockList")
if (blocks[address]) {
block = blocks[address]
} else {
var includes = await this.cmdp("filterIncludeList", {all_sites: true, filters: true})
for (let include of includes) {
if (include["siteblocks"][address]) {
var block = include["siteblocks"][address]
block["include"] = include
}
}
}
this.blocks = blocks
var reason = block["reason"]
if (!reason) reason = "Unknown reason"
var date = new Date(block["date_added"] * 1000)
document.getElementById("reason").innerText = reason
document.getElementById("added").innerText = "at " + date.toLocaleDateString() + " " + date.toLocaleTimeString()
if (block["include"]) {
document.getElementById("added").innerText += " from a shared blocklist"
document.getElementById("button").innerText = "Ignore blocking and visit the site"
}
document.getElementById("details").style.transform = "scale(1) rotateX(0deg)"
document.getElementById("button").style.transform = "translateY(0)"
document.getElementById("button").style.opacity = "1"
document.getElementById("button").onclick = () => {
if (block["include"])
this.cmd("siteAdd", address, () => { this.cmd("wrapperReload") })
else
this.cmd("siteblockRemove", address, () => { this.cmd("wrapperReload") })
}
}
}
page = new Page()
</script>
</body>
</html>

View File

@ -0,0 +1,119 @@
// Version 1.0.0 - Initial release
// Version 1.1.0 (2017-08-02) - Added cmdp function that returns promise instead of using callback
// Version 1.2.0 (2017-08-02) - Added Ajax monkey patch to emulate XMLHttpRequest over ZeroFrame API
const CMD_INNER_READY = 'innerReady'
const CMD_RESPONSE = 'response'
const CMD_WRAPPER_READY = 'wrapperReady'
const CMD_PING = 'ping'
const CMD_PONG = 'pong'
const CMD_WRAPPER_OPENED_WEBSOCKET = 'wrapperOpenedWebsocket'
const CMD_WRAPPER_CLOSE_WEBSOCKET = 'wrapperClosedWebsocket'
class ZeroFrame {
constructor(url) {
this.url = url
this.waiting_cb = {}
this.wrapper_nonce = document.location.href.replace(/.*wrapper_nonce=([A-Za-z0-9]+).*/, "$1")
this.connect()
this.next_message_id = 1
this.init()
}
init() {
return this
}
connect() {
this.target = window.parent
window.addEventListener('message', e => this.onMessage(e), false)
this.cmd(CMD_INNER_READY)
}
onMessage(e) {
let message = e.data
let cmd = message.cmd
if (cmd === CMD_RESPONSE) {
if (this.waiting_cb[message.to] !== undefined) {
this.waiting_cb[message.to](message.result)
}
else {
this.log("Websocket callback not found:", message)
}
} else if (cmd === CMD_WRAPPER_READY) {
this.cmd(CMD_INNER_READY)
} else if (cmd === CMD_PING) {
this.response(message.id, CMD_PONG)
} else if (cmd === CMD_WRAPPER_OPENED_WEBSOCKET) {
this.onOpenWebsocket()
} else if (cmd === CMD_WRAPPER_CLOSE_WEBSOCKET) {
this.onCloseWebsocket()
} else {
this.onRequest(cmd, message)
}
}
onRequest(cmd, message) {
this.log("Unknown request", message)
}
response(to, result) {
this.send({
cmd: CMD_RESPONSE,
to: to,
result: result
})
}
cmd(cmd, params={}, cb=null) {
this.send({
cmd: cmd,
params: params
}, cb)
}
cmdp(cmd, params={}) {
return new Promise((resolve, reject) => {
this.cmd(cmd, params, (res) => {
if (res && res.error) {
reject(res.error)
} else {
resolve(res)
}
})
})
}
send(message, cb=null) {
message.wrapper_nonce = this.wrapper_nonce
message.id = this.next_message_id
this.next_message_id++
this.target.postMessage(message, '*')
if (cb) {
this.waiting_cb[message.id] = cb
}
}
log(...args) {
console.log.apply(console, ['[ZeroFrame]'].concat(args))
}
onOpenWebsocket() {
this.log('Websocket open')
}
onCloseWebsocket() {
this.log('Websocket close')
}
monkeyPatchAjax() {
var page = this
XMLHttpRequest.prototype.realOpen = XMLHttpRequest.prototype.open
this.cmd("wrapperGetAjaxKey", [], (res) => { this.ajax_key = res })
var newOpen = function (method, url, async) {
url += "?ajax_key=" + page.ajax_key
return this.realOpen(method, url, async)
}
XMLHttpRequest.prototype.open = newOpen
}
}