2016-04-18 00:47:09 +02:00
|
|
|
import time
|
2016-08-20 10:39:15 +02:00
|
|
|
import re
|
2016-04-18 00:47:09 +02:00
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
from Plugin import PluginManager
|
2019-04-15 12:31:33 +02:00
|
|
|
from Db.DbQuery import DbQuery
|
2017-12-11 19:21:17 +01:00
|
|
|
from Debug import Debug
|
2019-04-29 16:44:13 +02:00
|
|
|
from util import helper
|
2019-08-26 03:20:07 +02:00
|
|
|
from util.Flag import flag
|
2016-04-18 00:47:09 +02:00
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
|
|
|
|
@PluginManager.registerTo("UiWebsocket")
|
|
|
|
class UiWebsocketPlugin(object):
|
2017-02-14 19:03:17 +01:00
|
|
|
def formatSiteInfo(self, site, create_user=True):
|
2018-04-18 22:06:00 +02:00
|
|
|
site_info = super(UiWebsocketPlugin, self).formatSiteInfo(site, create_user=create_user)
|
2018-04-18 22:06:18 +02:00
|
|
|
feed_following = self.user.sites.get(site.address, {}).get("follow", None)
|
2017-02-14 19:03:17 +01:00
|
|
|
if feed_following == None:
|
2017-02-14 19:04:25 +01:00
|
|
|
site_info["feed_follow_num"] = None
|
2017-02-14 19:03:17 +01:00
|
|
|
else:
|
2017-02-14 19:04:25 +01:00
|
|
|
site_info["feed_follow_num"] = len(feed_following)
|
2017-02-14 19:03:17 +01:00
|
|
|
return site_info
|
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
def actionFeedFollow(self, to, feeds):
|
|
|
|
self.user.setFeedFollow(self.site.address, feeds)
|
2016-03-11 13:26:54 +01:00
|
|
|
self.user.save()
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
self.response(to, "ok")
|
|
|
|
|
|
|
|
def actionFeedListFollow(self, to):
|
2019-07-01 16:08:21 +02:00
|
|
|
feeds = self.user.sites.get(self.site.address, {}).get("follow", {})
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
self.response(to, feeds)
|
|
|
|
|
2019-08-26 03:20:07 +02:00
|
|
|
@flag.admin
|
2017-01-29 19:02:26 +01:00
|
|
|
def actionFeedQuery(self, to, limit=10, day_limit=3):
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
from Site import SiteManager
|
|
|
|
rows = []
|
2017-12-11 19:21:17 +01:00
|
|
|
stats = []
|
|
|
|
|
|
|
|
total_s = time.time()
|
|
|
|
num_sites = 0
|
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
for address, site_data in list(self.user.sites.items()):
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
feeds = site_data.get("follow")
|
|
|
|
if not feeds:
|
|
|
|
continue
|
2017-02-22 11:28:48 +01:00
|
|
|
if type(feeds) is not dict:
|
|
|
|
self.log.debug("Invalid feed for site %s" % address)
|
|
|
|
continue
|
2017-12-11 19:21:17 +01:00
|
|
|
num_sites += 1
|
2019-03-15 21:06:59 +01:00
|
|
|
for name, query_set in feeds.items():
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
site = SiteManager.site_manager.get(address)
|
2017-12-27 11:09:16 +01:00
|
|
|
if not site or not site.storage.has_db:
|
2017-12-11 19:35:27 +01:00
|
|
|
continue
|
|
|
|
|
2017-12-11 19:21:17 +01:00
|
|
|
s = time.time()
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
try:
|
2017-12-12 23:46:05 +01:00
|
|
|
query_raw, params = query_set
|
2017-12-14 20:13:26 +01:00
|
|
|
query_parts = re.split(r"UNION(?:\s+ALL|)", query_raw)
|
2016-08-16 19:57:41 +02:00
|
|
|
for i, query_part in enumerate(query_parts):
|
|
|
|
db_query = DbQuery(query_part)
|
2017-01-29 19:02:26 +01:00
|
|
|
if day_limit:
|
|
|
|
where = " WHERE %s > strftime('%%s', 'now', '-%s day')" % (db_query.fields.get("date_added", "date_added"), day_limit)
|
|
|
|
if "WHERE" in query_part:
|
|
|
|
query_part = re.sub("WHERE (.*?)(?=$| GROUP BY)", where+" AND (\\1)", query_part)
|
|
|
|
else:
|
|
|
|
query_part += where
|
2016-08-16 19:57:41 +02:00
|
|
|
query_parts[i] = query_part
|
|
|
|
query = " UNION ".join(query_parts)
|
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
if ":params" in query:
|
2019-04-29 16:44:13 +02:00
|
|
|
query_params = map(helper.sqlquote, params)
|
|
|
|
query = query.replace(":params", ",".join(query_params))
|
|
|
|
|
|
|
|
res = site.storage.query(query + " ORDER BY date_added DESC LIMIT %s" % limit)
|
2017-02-22 11:29:07 +01:00
|
|
|
|
2017-07-16 22:43:57 +02:00
|
|
|
except Exception as err: # Log error
|
2017-12-11 19:21:17 +01:00
|
|
|
self.log.error("%s feed query %s error: %s" % (address, name, Debug.formatException(err)))
|
2019-04-29 16:44:13 +02:00
|
|
|
stats.append({"site": site.address, "feed_name": name, "error": str(err)})
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
continue
|
2017-02-22 11:29:07 +01:00
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
for row in res:
|
|
|
|
row = dict(row)
|
2019-03-15 21:06:59 +01:00
|
|
|
if not isinstance(row["date_added"], (int, float, complex)):
|
2018-05-30 17:16:45 +02:00
|
|
|
self.log.debug("Invalid date_added from site %s: %r" % (address, row["date_added"]))
|
|
|
|
continue
|
2017-02-22 11:29:07 +01:00
|
|
|
if row["date_added"] > 1000000000000: # Formatted as millseconds
|
|
|
|
row["date_added"] = row["date_added"] / 1000
|
2016-05-16 22:24:30 +02:00
|
|
|
if "date_added" not in row or row["date_added"] > time.time() + 120:
|
2018-05-30 17:16:45 +02:00
|
|
|
self.log.debug("Newsfeed item from the future from from site %s" % address)
|
2016-03-03 21:12:16 +01:00
|
|
|
continue # Feed item is in the future, skip it
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
row["site"] = address
|
|
|
|
row["feed_name"] = name
|
|
|
|
rows.append(row)
|
2017-12-11 19:21:17 +01:00
|
|
|
stats.append({"site": site.address, "feed_name": name, "taken": round(time.time() - s, 3)})
|
2019-04-08 18:12:00 +02:00
|
|
|
time.sleep(0.001)
|
2017-12-11 19:21:17 +01:00
|
|
|
return self.response(to, {"rows": rows, "stats": stats, "num": len(rows), "sites": num_sites, "taken": round(time.time() - total_s, 3)})
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
|
2017-11-10 11:15:01 +01:00
|
|
|
def parseSearch(self, search):
|
|
|
|
parts = re.split("(site|type):", search)
|
|
|
|
if len(parts) > 1: # Found filter
|
|
|
|
search_text = parts[0]
|
|
|
|
parts = [part.strip() for part in parts]
|
|
|
|
filters = dict(zip(parts[1::2], parts[2::2]))
|
|
|
|
else:
|
|
|
|
search_text = search
|
|
|
|
filters = {}
|
|
|
|
return [search_text, filters]
|
|
|
|
|
2019-01-25 14:26:38 +01:00
|
|
|
def actionFeedSearch(self, to, search, limit=30, day_limit=30):
|
2016-04-18 00:47:09 +02:00
|
|
|
if "ADMIN" not in self.site.settings["permissions"]:
|
|
|
|
return self.response(to, "FeedSearch not allowed")
|
|
|
|
|
|
|
|
from Site import SiteManager
|
|
|
|
rows = []
|
2017-12-11 19:21:17 +01:00
|
|
|
stats = []
|
2016-04-18 00:47:09 +02:00
|
|
|
num_sites = 0
|
2017-12-11 19:21:17 +01:00
|
|
|
total_s = time.time()
|
2017-11-10 11:15:01 +01:00
|
|
|
|
|
|
|
search_text, filters = self.parseSearch(search)
|
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
for address, site in SiteManager.site_manager.list().items():
|
2016-04-18 00:47:09 +02:00
|
|
|
if not site.storage.has_db:
|
|
|
|
continue
|
|
|
|
|
2017-11-10 11:15:10 +01:00
|
|
|
if "site" in filters:
|
|
|
|
if filters["site"].lower() not in [site.address, site.content_manager.contents["content.json"].get("title").lower()]:
|
|
|
|
continue
|
|
|
|
|
2016-04-19 12:00:58 +02:00
|
|
|
if site.storage.db: # Database loaded
|
|
|
|
feeds = site.storage.db.schema.get("feeds")
|
|
|
|
else:
|
|
|
|
try:
|
|
|
|
feeds = site.storage.loadJson("dbschema.json").get("feeds")
|
|
|
|
except:
|
|
|
|
continue
|
2016-04-18 00:47:09 +02:00
|
|
|
|
|
|
|
if not feeds:
|
|
|
|
continue
|
|
|
|
|
|
|
|
num_sites += 1
|
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
for name, query in feeds.items():
|
2017-12-11 19:21:17 +01:00
|
|
|
s = time.time()
|
2016-04-18 00:47:09 +02:00
|
|
|
try:
|
|
|
|
db_query = DbQuery(query)
|
2017-11-10 11:15:57 +01:00
|
|
|
|
|
|
|
params = []
|
|
|
|
# Filters
|
|
|
|
if search_text:
|
|
|
|
db_query.wheres.append("(%s LIKE ? OR %s LIKE ?)" % (db_query.fields["body"], db_query.fields["title"]))
|
|
|
|
search_like = "%" + search_text.replace(" ", "%") + "%"
|
|
|
|
params.append(search_like)
|
|
|
|
params.append(search_like)
|
2017-11-10 11:16:08 +01:00
|
|
|
if filters.get("type") and filters["type"] not in query:
|
|
|
|
continue
|
2017-11-10 11:15:57 +01:00
|
|
|
|
2019-01-25 14:26:38 +01:00
|
|
|
if day_limit:
|
|
|
|
db_query.wheres.append(
|
|
|
|
"%s > strftime('%%s', 'now', '-%s day')" % (db_query.fields.get("date_added", "date_added"), day_limit)
|
|
|
|
)
|
|
|
|
|
2017-11-10 11:15:57 +01:00
|
|
|
# Order
|
2016-04-18 00:47:09 +02:00
|
|
|
db_query.parts["ORDER BY"] = "date_added DESC"
|
2019-01-25 14:26:38 +01:00
|
|
|
db_query.parts["LIMIT"] = str(limit)
|
2016-04-18 00:47:09 +02:00
|
|
|
|
2017-11-10 11:15:57 +01:00
|
|
|
res = site.storage.query(str(db_query), params)
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2017-12-11 19:21:17 +01:00
|
|
|
self.log.error("%s feed query %s error: %s" % (address, name, Debug.formatException(err)))
|
|
|
|
stats.append({"site": site.address, "feed_name": name, "error": str(err), "query": query})
|
2016-04-18 00:47:09 +02:00
|
|
|
continue
|
|
|
|
for row in res:
|
|
|
|
row = dict(row)
|
2019-04-11 01:18:52 +02:00
|
|
|
if not row["date_added"] or row["date_added"] > time.time() + 120:
|
2016-04-18 00:47:09 +02:00
|
|
|
continue # Feed item is in the future, skip it
|
|
|
|
row["site"] = address
|
|
|
|
row["feed_name"] = name
|
|
|
|
rows.append(row)
|
2017-12-11 19:21:17 +01:00
|
|
|
stats.append({"site": site.address, "feed_name": name, "taken": round(time.time() - s, 3)})
|
|
|
|
return self.response(to, {"rows": rows, "num": len(rows), "sites": num_sites, "taken": round(time.time() - total_s, 3), "stats": stats})
|
2016-04-18 00:47:09 +02:00
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
|
|
|
|
@PluginManager.registerTo("User")
|
|
|
|
class UserPlugin(object):
|
|
|
|
# Set queries that user follows
|
|
|
|
def setFeedFollow(self, address, feeds):
|
|
|
|
site_data = self.getSiteData(address)
|
|
|
|
site_data["follow"] = feeds
|
|
|
|
self.save()
|
|
|
|
return site_data
|