2015-07-12 20:36:46 +02:00
|
|
|
import time
|
2019-03-16 02:27:04 +01:00
|
|
|
import html
|
2015-07-12 20:36:46 +02:00
|
|
|
import os
|
2018-08-29 19:58:18 +02:00
|
|
|
import json
|
2019-03-23 03:40:42 +01:00
|
|
|
from collections import OrderedDict
|
2015-07-12 20:36:46 +02:00
|
|
|
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
from Plugin import PluginManager
|
2015-04-12 23:59:22 +02:00
|
|
|
from Config import config
|
2019-03-16 02:27:04 +01:00
|
|
|
from util import helper
|
2019-03-23 03:40:42 +01:00
|
|
|
from Debug import Debug
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
@PluginManager.registerTo("UiRequest")
|
|
|
|
class UiRequestPlugin(object):
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-02-27 00:00:27 +01:00
|
|
|
def formatTableRow(self, row, class_name=""):
|
2015-07-12 20:36:46 +02:00
|
|
|
back = []
|
|
|
|
for format, val in row:
|
|
|
|
if val is None:
|
|
|
|
formatted = "n/a"
|
|
|
|
elif format == "since":
|
|
|
|
if val:
|
|
|
|
formatted = "%.0f" % (time.time() - val)
|
|
|
|
else:
|
|
|
|
formatted = "n/a"
|
|
|
|
else:
|
|
|
|
formatted = format % val
|
|
|
|
back.append("<td>%s</td>" % formatted)
|
2019-03-16 02:27:04 +01:00
|
|
|
return "<tr class='%s'>%s</tr>" % (class_name, "".join(back))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
def getObjSize(self, obj, hpy=None):
|
|
|
|
if hpy:
|
|
|
|
return float(hpy.iso(obj).domisize) / 1024
|
|
|
|
else:
|
|
|
|
return 0
|
|
|
|
|
|
|
|
# /Stats entry point
|
2019-03-16 02:27:04 +01:00
|
|
|
@helper.encodeResponse
|
2015-07-12 20:36:46 +02:00
|
|
|
def actionStats(self):
|
|
|
|
import gc
|
|
|
|
import sys
|
|
|
|
from Ui import UiRequest
|
2016-09-07 17:42:40 +02:00
|
|
|
from Db import Db
|
2015-07-12 20:36:46 +02:00
|
|
|
from Crypt import CryptConnection
|
|
|
|
|
2019-01-31 00:39:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
hpy = None
|
|
|
|
if self.get.get("size") == "1": # Calc obj size
|
|
|
|
try:
|
|
|
|
import guppy
|
|
|
|
hpy = guppy.hpy()
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
self.sendHeader()
|
2016-03-16 00:37:31 +01:00
|
|
|
|
|
|
|
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
|
|
|
|
yield "This function is disabled on this proxy"
|
|
|
|
raise StopIteration
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
s = time.time()
|
|
|
|
main = sys.modules["main"]
|
|
|
|
|
|
|
|
# Style
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace }
|
|
|
|
table td, table th { text-align: right; padding: 0px 10px }
|
2017-02-27 00:00:27 +01:00
|
|
|
.connections td { white-space: nowrap }
|
|
|
|
.serving-False { opacity: 0.3 }
|
2015-07-12 20:36:46 +02:00
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
|
|
|
# Memory
|
2018-08-29 19:56:27 +02:00
|
|
|
yield "rev%s | " % config.rev
|
2019-01-26 20:42:27 +01:00
|
|
|
yield "%s | " % main.file_server.ip_external_list
|
2018-08-29 19:56:27 +02:00
|
|
|
yield "Port: %s | " % main.file_server.port
|
2019-01-23 02:13:29 +01:00
|
|
|
yield "IP Network: %s | " % main.file_server.supported_ip_types
|
2018-08-29 19:56:27 +02:00
|
|
|
yield "Opened: %s | " % main.file_server.port_opened
|
|
|
|
yield "Crypt: %s | " % CryptConnection.manager.crypt_supported
|
|
|
|
yield "In: %.2fMB, Out: %.2fMB | " % (
|
|
|
|
float(main.file_server.bytes_recv) / 1024 / 1024,
|
|
|
|
float(main.file_server.bytes_sent) / 1024 / 1024
|
|
|
|
)
|
|
|
|
yield "Peerid: %s | " % main.file_server.peer_id
|
2018-09-05 14:32:01 +02:00
|
|
|
yield "Time correction: %.2fs" % main.file_server.getTimecorrection()
|
2018-08-29 19:56:27 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
try:
|
|
|
|
import psutil
|
|
|
|
process = psutil.Process(os.getpid())
|
|
|
|
mem = process.get_memory_info()[0] / float(2 ** 20)
|
|
|
|
yield "Mem: %.2fMB | " % mem
|
|
|
|
yield "Threads: %s | " % len(process.threads())
|
|
|
|
yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
|
|
|
|
yield "Files: %s | " % len(process.open_files())
|
|
|
|
yield "Sockets: %s | " % len(process.connections())
|
|
|
|
yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
|
|
|
|
except Exception:
|
|
|
|
pass
|
|
|
|
yield "<br>"
|
|
|
|
|
|
|
|
# Connections
|
2018-09-02 02:19:51 +02:00
|
|
|
yield "<b>Connections</b> (%s, total made: %s, in: %s, out: %s):<br>" % (
|
|
|
|
len(main.file_server.connections), main.file_server.last_connection_id, main.file_server.num_incoming, main.file_server.num_outgoing
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
2017-02-27 00:00:27 +01:00
|
|
|
yield "<table class='connections'><tr> <th>id</th> <th>type</th> <th>ip</th> <th>open</th> <th>crypt</th> <th>ping</th>"
|
2016-11-07 23:00:17 +01:00
|
|
|
yield "<th>buff</th> <th>bad</th> <th>idle</th> <th>open</th> <th>delay</th> <th>cpu</th> <th>out</th> <th>in</th> <th>last sent</th>"
|
2018-08-29 19:57:38 +02:00
|
|
|
yield "<th>wait</th> <th>version</th> <th>time</th> <th>sites</th> </tr>"
|
2015-07-12 20:36:46 +02:00
|
|
|
for connection in main.file_server.connections:
|
|
|
|
if "cipher" in dir(connection.sock):
|
|
|
|
cipher = connection.sock.cipher()[0]
|
2018-10-05 15:08:28 +02:00
|
|
|
tls_version = connection.sock.version()
|
2015-07-12 20:36:46 +02:00
|
|
|
else:
|
|
|
|
cipher = connection.crypt
|
2018-10-05 15:08:28 +02:00
|
|
|
tls_version = ""
|
2018-08-29 19:57:38 +02:00
|
|
|
if "time" in connection.handshake and connection.last_ping_delay:
|
|
|
|
time_correction = connection.handshake["time"] - connection.handshake_time - connection.last_ping_delay
|
|
|
|
else:
|
|
|
|
time_correction = 0.0
|
2015-07-12 20:36:46 +02:00
|
|
|
yield self.formatTableRow([
|
|
|
|
("%3d", connection.id),
|
|
|
|
("%s", connection.type),
|
|
|
|
("%s:%s", (connection.ip, connection.port)),
|
|
|
|
("%s", connection.handshake.get("port_opened")),
|
2018-10-05 15:08:28 +02:00
|
|
|
("<span title='%s %s'>%s</span>", (cipher, tls_version, connection.crypt)),
|
2015-07-12 20:36:46 +02:00
|
|
|
("%6.3f", connection.last_ping_delay),
|
|
|
|
("%s", connection.incomplete_buff_recv),
|
2016-03-12 23:09:26 +01:00
|
|
|
("%s", connection.bad_actions),
|
2015-07-12 20:36:46 +02:00
|
|
|
("since", max(connection.last_send_time, connection.last_recv_time)),
|
|
|
|
("since", connection.start_time),
|
2018-08-29 19:57:38 +02:00
|
|
|
("%.3f", max(-1, connection.last_sent_time - connection.last_send_time)),
|
2017-02-27 00:00:27 +01:00
|
|
|
("%.3f", connection.cpu_time),
|
2019-03-16 02:27:04 +01:00
|
|
|
("%.0fk", connection.bytes_sent / 1024),
|
|
|
|
("%.0fk", connection.bytes_recv / 1024),
|
2017-10-13 01:16:44 +02:00
|
|
|
("<span title='Recv: %s'>%s</span>", (connection.last_cmd_recv, connection.last_cmd_sent)),
|
2019-03-16 02:27:04 +01:00
|
|
|
("%s", list(connection.waiting_requests.keys())),
|
2015-07-12 20:36:46 +02:00
|
|
|
("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?"))),
|
2018-08-29 19:57:38 +02:00
|
|
|
("%.2fs", time_correction),
|
2016-04-25 02:23:19 +02:00
|
|
|
("%s", connection.sites)
|
2015-07-12 20:36:46 +02:00
|
|
|
])
|
|
|
|
yield "</table>"
|
|
|
|
|
2018-08-26 22:53:01 +02:00
|
|
|
# Trackers
|
|
|
|
yield "<br><br><b>Trackers:</b><br>"
|
|
|
|
yield "<table class='trackers'><tr> <th>address</th> <th>request</th> <th>successive errors</th> <th>last_request</th></tr>"
|
2019-03-16 02:27:04 +01:00
|
|
|
for tracker_address, tracker_stat in sorted(sys.modules["Site.SiteAnnouncer"].global_stats.items()):
|
2018-08-26 22:53:01 +02:00
|
|
|
yield self.formatTableRow([
|
|
|
|
("%s", tracker_address),
|
|
|
|
("%s", tracker_stat["num_request"]),
|
|
|
|
("%s", tracker_stat["num_error"]),
|
|
|
|
("%.0f min ago", min(999, (time.time() - tracker_stat["time_request"]) / 60))
|
|
|
|
])
|
|
|
|
yield "</table>"
|
|
|
|
|
|
|
|
if "AnnounceShare" in PluginManager.plugin_manager.plugin_names:
|
|
|
|
yield "<br><br><b>Shared trackers:</b><br>"
|
2018-08-29 19:57:54 +02:00
|
|
|
yield "<table class='trackers'><tr> <th>address</th> <th>added</th> <th>found</th> <th>latency</th> <th>successive errors</th> <th>last_success</th></tr>"
|
2018-08-26 22:53:01 +02:00
|
|
|
from AnnounceShare import AnnounceSharePlugin
|
2019-03-16 02:27:04 +01:00
|
|
|
for tracker_address, tracker_stat in sorted(AnnounceSharePlugin.tracker_storage.getTrackers().items()):
|
2018-08-26 22:53:01 +02:00
|
|
|
yield self.formatTableRow([
|
|
|
|
("%s", tracker_address),
|
|
|
|
("%.0f min ago", min(999, (time.time() - tracker_stat["time_added"]) / 60)),
|
2018-08-29 19:57:54 +02:00
|
|
|
("%.0f min ago", min(999, (time.time() - tracker_stat.get("time_found", 0)) / 60)),
|
2018-08-26 22:53:01 +02:00
|
|
|
("%.3fs", tracker_stat["latency"]),
|
|
|
|
("%s", tracker_stat["num_error"]),
|
|
|
|
("%.0f min ago", min(999, (time.time() - tracker_stat["time_success"]) / 60)),
|
|
|
|
])
|
|
|
|
yield "</table>"
|
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# Tor hidden services
|
2019-03-16 02:27:04 +01:00
|
|
|
yield "<br><br><b>Tor hidden services (status: %s):</b><br>" % main.file_server.tor_manager.status
|
|
|
|
for site_address, onion in list(main.file_server.tor_manager.site_onions.items()):
|
|
|
|
yield "- %-34s: %s<br>" % (site_address, onion)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
2016-09-07 17:42:40 +02:00
|
|
|
# Db
|
|
|
|
yield "<br><br><b>Db</b>:<br>"
|
|
|
|
for db in sys.modules["Db.Db"].opened_dbs:
|
2018-08-29 19:58:18 +02:00
|
|
|
tables = [row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()]
|
|
|
|
table_rows = {}
|
|
|
|
for table in tables:
|
|
|
|
table_rows[table] = db.execute("SELECT COUNT(*) AS c FROM %s" % table).fetchone()["c"]
|
|
|
|
db_size = os.path.getsize(db.db_path) / 1024.0 / 1024.0
|
|
|
|
yield "- %.3fs: %s %.3fMB, table rows: %s<br>" % (
|
2019-03-16 02:27:04 +01:00
|
|
|
time.time() - db.last_query_time, db.db_path, db_size, json.dumps(table_rows, sort_keys=True)
|
2018-08-29 19:58:18 +02:00
|
|
|
)
|
|
|
|
|
2016-09-07 17:42:40 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Sites
|
|
|
|
yield "<br><br><b>Sites</b>:"
|
|
|
|
yield "<table>"
|
2015-08-16 11:51:00 +02:00
|
|
|
yield "<tr><th>address</th> <th>connected</th> <th title='connected/good/total'>peers</th> <th>content.json</th> <th>out</th> <th>in</th> </tr>"
|
2019-03-16 02:27:04 +01:00
|
|
|
for site in list(self.server.sites.values()):
|
2015-07-12 20:36:46 +02:00
|
|
|
yield self.formatTableRow([
|
|
|
|
(
|
2017-02-27 00:00:27 +01:00
|
|
|
"""<a href='#' onclick='document.getElementById("peers_%s").style.display="initial"; return false'>%s</a>""",
|
|
|
|
(site.address, site.address)
|
2015-07-12 20:36:46 +02:00
|
|
|
),
|
2019-03-16 02:27:04 +01:00
|
|
|
("%s", [peer.connection.id for peer in list(site.peers.values()) if peer.connection and peer.connection.connected]),
|
2015-07-12 20:36:46 +02:00
|
|
|
("%s/%s/%s", (
|
2019-03-16 02:27:04 +01:00
|
|
|
len([peer for peer in list(site.peers.values()) if peer.connection and peer.connection.connected]),
|
2015-07-12 20:36:46 +02:00
|
|
|
len(site.getConnectablePeers(100)),
|
|
|
|
len(site.peers)
|
|
|
|
)),
|
2016-09-07 17:42:50 +02:00
|
|
|
("%s (loaded: %s)", (
|
|
|
|
len(site.content_manager.contents),
|
2019-03-16 02:27:04 +01:00
|
|
|
len([key for key, val in dict(site.content_manager.contents).items() if val])
|
2016-09-07 17:42:50 +02:00
|
|
|
)),
|
2019-03-16 02:27:04 +01:00
|
|
|
("%.0fk", site.settings.get("bytes_sent", 0) / 1024),
|
|
|
|
("%.0fk", site.settings.get("bytes_recv", 0) / 1024),
|
2017-02-27 00:00:27 +01:00
|
|
|
], "serving-%s" % site.settings["serving"])
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
yield "<tr><td id='peers_%s' style='display: none; white-space: pre' colspan=6>" % site.address
|
2019-03-16 02:27:04 +01:00
|
|
|
for key, peer in list(site.peers.items()):
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
if peer.time_found:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
time_found = int(time.time() - peer.time_found) / 60
|
2015-07-25 13:38:58 +02:00
|
|
|
else:
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
time_found = "--"
|
2015-07-25 13:38:58 +02:00
|
|
|
if peer.connection:
|
|
|
|
connection_id = peer.connection.id
|
|
|
|
else:
|
|
|
|
connection_id = None
|
2017-10-03 14:36:50 +02:00
|
|
|
if site.content_manager.has_optional_files:
|
2015-11-17 12:48:03 +01:00
|
|
|
yield "Optional files: %4s " % len(peer.hashfield)
|
2017-02-27 00:00:27 +01:00
|
|
|
time_added = (time.time() - peer.time_added) / (60 * 60 * 24)
|
2018-09-05 14:31:26 +02:00
|
|
|
yield "(#%4s, rep: %2s, err: %s, found: %3s min, add: %.1f day) %30s -<br>" % (connection_id, peer.reputation, peer.connection_error, time_found, time_added, key)
|
2015-07-12 20:36:46 +02:00
|
|
|
yield "<br></td></tr>"
|
|
|
|
yield "</table>"
|
|
|
|
|
2017-10-13 01:17:07 +02:00
|
|
|
# Big files
|
|
|
|
yield "<br><br><b>Big files</b>:<br>"
|
2019-03-16 02:27:04 +01:00
|
|
|
for site in list(self.server.sites.values()):
|
2017-10-13 01:17:07 +02:00
|
|
|
if not site.settings.get("has_bigfile"):
|
|
|
|
continue
|
|
|
|
bigfiles = {}
|
|
|
|
yield """<a href="#" onclick='document.getElementById("bigfiles_%s").style.display="initial"; return false'>%s</a><br>""" % (site.address, site.address)
|
2019-03-16 02:27:04 +01:00
|
|
|
for peer in list(site.peers.values()):
|
2017-10-13 01:17:07 +02:00
|
|
|
if not peer.time_piecefields_updated:
|
|
|
|
continue
|
2019-03-16 02:27:04 +01:00
|
|
|
for sha512, piecefield in peer.piecefields.items():
|
2017-10-13 01:17:07 +02:00
|
|
|
if sha512 not in bigfiles:
|
|
|
|
bigfiles[sha512] = []
|
|
|
|
bigfiles[sha512].append(peer)
|
|
|
|
|
|
|
|
yield "<div id='bigfiles_%s' style='display: none'>" % site.address
|
2019-03-16 02:27:04 +01:00
|
|
|
for sha512, peers in bigfiles.items():
|
2017-10-16 15:12:17 +02:00
|
|
|
yield "<br> - " + sha512 + " (hash id: %s)<br>" % site.content_manager.hashfield.getHashId(sha512)
|
2017-10-13 01:17:07 +02:00
|
|
|
yield "<table>"
|
|
|
|
for peer in peers:
|
|
|
|
yield "<tr><td>" + peer.key + "</td><td>" + peer.piecefields[sha512].tostring() + "</td></tr>"
|
|
|
|
yield "</table>"
|
|
|
|
yield "</div>"
|
|
|
|
|
2017-10-13 01:17:27 +02:00
|
|
|
# Cmd stats
|
|
|
|
yield "<div style='float: left'>"
|
|
|
|
yield "<br><br><b>Sent commands</b>:<br>"
|
|
|
|
yield "<table>"
|
2019-03-16 02:27:04 +01:00
|
|
|
for stat_key, stat in sorted(main.file_server.stat_sent.items(), key=lambda i: i[1]["bytes"], reverse=True):
|
2017-10-16 15:12:17 +02:00
|
|
|
yield "<tr><td>%s</td><td style='white-space: nowrap'>x %s =</td><td>%.0fkB</td></tr>" % (stat_key, stat["num"], stat["bytes"] / 1024)
|
2017-10-13 01:17:27 +02:00
|
|
|
yield "</table>"
|
|
|
|
yield "</div>"
|
|
|
|
|
2017-10-16 15:12:17 +02:00
|
|
|
yield "<div style='float: left; margin-left: 20%; max-width: 50%'>"
|
2017-10-13 01:17:27 +02:00
|
|
|
yield "<br><br><b>Received commands</b>:<br>"
|
|
|
|
yield "<table>"
|
2019-03-16 02:27:04 +01:00
|
|
|
for stat_key, stat in sorted(main.file_server.stat_recv.items(), key=lambda i: i[1]["bytes"], reverse=True):
|
2017-10-16 15:12:17 +02:00
|
|
|
yield "<tr><td>%s</td><td style='white-space: nowrap'>x %s =</td><td>%.0fkB</td></tr>" % (stat_key, stat["num"], stat["bytes"] / 1024)
|
2017-10-13 01:17:27 +02:00
|
|
|
yield "</table>"
|
|
|
|
yield "</div>"
|
|
|
|
yield "<div style='clear: both'></div>"
|
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
# No more if not in debug mode
|
|
|
|
if not config.debug:
|
|
|
|
raise StopIteration
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Object types
|
|
|
|
|
|
|
|
obj_count = {}
|
|
|
|
for obj in gc.get_objects():
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if obj_type not in obj_count:
|
|
|
|
obj_count[obj_type] = [0, 0]
|
|
|
|
obj_count[obj_type][0] += 1 # Count
|
|
|
|
obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
|
|
|
|
|
|
|
|
yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
|
|
|
|
len(obj_count),
|
2019-03-16 02:27:04 +01:00
|
|
|
sum([stat[0] for stat in list(obj_count.values())]),
|
|
|
|
sum([stat[1] for stat in list(obj_count.values())])
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
for obj, stat in sorted(list(obj_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count
|
|
|
|
yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, html.escape(obj))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Classes
|
|
|
|
|
|
|
|
class_count = {}
|
|
|
|
for obj in gc.get_objects():
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if obj_type != "<type 'instance'>":
|
|
|
|
continue
|
|
|
|
class_name = obj.__class__.__name__
|
|
|
|
if class_name not in class_count:
|
|
|
|
class_count[class_name] = [0, 0]
|
|
|
|
class_count[class_name][0] += 1 # Count
|
|
|
|
class_count[class_name][1] += float(sys.getsizeof(obj)) / 1024 # Size
|
|
|
|
|
|
|
|
yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (
|
|
|
|
len(class_count),
|
2019-03-16 02:27:04 +01:00
|
|
|
sum([stat[0] for stat in list(class_count.values())]),
|
|
|
|
sum([stat[1] for stat in list(class_count.values())])
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
for obj, stat in sorted(list(class_count.items()), key=lambda x: x[1][0], reverse=True): # Sorted by count
|
|
|
|
yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, html.escape(obj))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
from greenlet import greenlet
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
|
|
|
|
yield "<br>Greenlets (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
from Worker import Worker
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
|
|
|
|
yield "<br>Workers (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
from Connection import Connection
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
|
|
|
|
yield "<br>Connections (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
from socket import socket
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
|
|
|
|
yield "<br>Sockets (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
from msgpack import Unpacker
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
|
|
|
|
yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
from Site import Site
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
|
|
|
|
yield "<br>Sites (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
|
|
|
|
yield "<br>Loggers (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj.name)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
|
|
|
|
yield "<br>UiRequests (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
from Peer import Peer
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
|
|
|
|
yield "<br>Peers (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), html.escape(repr(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
objs = [(key, val) for key, val in sys.modules.items() if val is not None]
|
2015-07-12 20:36:46 +02:00
|
|
|
objs.sort()
|
|
|
|
yield "<br>Modules (%s):<br>" % len(objs)
|
|
|
|
for module_name, module in objs:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, html.escape(repr(module)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
gc.collect() # Implicit grabage collection
|
|
|
|
yield "Done in %.1f" % (time.time() - s)
|
|
|
|
|
|
|
|
def actionDumpobj(self):
|
2015-09-28 22:07:26 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
import gc
|
|
|
|
import sys
|
|
|
|
|
|
|
|
self.sendHeader()
|
2015-09-28 22:07:26 +02:00
|
|
|
|
2016-03-16 00:37:31 +01:00
|
|
|
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
|
|
|
|
yield "This function is disabled on this proxy"
|
|
|
|
raise StopIteration
|
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
# No more if not in debug mode
|
|
|
|
if not config.debug:
|
|
|
|
yield "Not in debug mode"
|
|
|
|
raise StopIteration
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
class_filter = self.get.get("class")
|
|
|
|
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace; white-space: pre }
|
|
|
|
table * { text-align: right; padding: 0px 10px }
|
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
|
|
|
objs = gc.get_objects()
|
|
|
|
for obj in objs:
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter:
|
|
|
|
continue
|
2019-03-16 02:27:04 +01:00
|
|
|
yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, html.escape(str(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
for attr in dir(obj):
|
2019-03-16 02:27:04 +01:00
|
|
|
yield "- %s: %s<br>" % (attr, html.escape(str(getattr(obj, attr))))
|
2015-07-12 20:36:46 +02:00
|
|
|
yield "<br>"
|
|
|
|
|
|
|
|
gc.collect() # Implicit grabage collection
|
|
|
|
|
|
|
|
def actionListobj(self):
|
2015-09-28 22:07:26 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
import gc
|
|
|
|
import sys
|
|
|
|
|
|
|
|
self.sendHeader()
|
2015-09-28 22:07:26 +02:00
|
|
|
|
2016-03-16 00:37:31 +01:00
|
|
|
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
|
|
|
|
yield "This function is disabled on this proxy"
|
|
|
|
raise StopIteration
|
|
|
|
|
2015-09-28 22:07:26 +02:00
|
|
|
# No more if not in debug mode
|
|
|
|
if not config.debug:
|
|
|
|
yield "Not in debug mode"
|
|
|
|
raise StopIteration
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
type_filter = self.get.get("type")
|
|
|
|
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace; white-space: pre }
|
|
|
|
table * { text-align: right; padding: 0px 10px }
|
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
yield "Listing all %s objects in memory...<br>" % html.escape(type_filter)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
ref_count = {}
|
|
|
|
objs = gc.get_objects()
|
|
|
|
for obj in objs:
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if obj_type != type_filter:
|
|
|
|
continue
|
|
|
|
refs = [
|
|
|
|
ref for ref in gc.get_referrers(obj)
|
|
|
|
if hasattr(ref, "__class__") and
|
|
|
|
ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"]
|
|
|
|
]
|
|
|
|
if not refs:
|
|
|
|
continue
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
try:
|
|
|
|
yield "%.1fkb <span title=\"%s\">%s</span>... " % (
|
2019-03-16 02:27:04 +01:00
|
|
|
float(sys.getsizeof(obj)) / 1024, html.escape(str(obj)), html.escape(str(obj)[0:100].ljust(100))
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
)
|
|
|
|
except:
|
|
|
|
continue
|
2015-07-12 20:36:46 +02:00
|
|
|
for ref in refs:
|
|
|
|
yield " ["
|
|
|
|
if "object at" in str(ref) or len(str(ref)) > 100:
|
|
|
|
yield str(ref.__class__.__name__)
|
|
|
|
else:
|
2019-03-16 02:27:04 +01:00
|
|
|
yield str(ref.__class__.__name__) + ":" + html.escape(str(ref))
|
2015-07-12 20:36:46 +02:00
|
|
|
yield "] "
|
|
|
|
ref_type = ref.__class__.__name__
|
|
|
|
if ref_type not in ref_count:
|
|
|
|
ref_count[ref_type] = [0, 0]
|
|
|
|
ref_count[ref_type][0] += 1 # Count
|
|
|
|
ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
|
|
|
|
yield "<br>"
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (len(ref_count), sum([stat[1] for stat in list(ref_count.values())]))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
for obj, stat in sorted(list(ref_count.items()), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count
|
|
|
|
yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], html.escape(str(obj)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
gc.collect() # Implicit grabage collection
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
@helper.encodeResponse
|
2015-07-12 20:36:46 +02:00
|
|
|
def actionBenchmark(self):
|
|
|
|
import sys
|
|
|
|
import gc
|
|
|
|
from contextlib import contextmanager
|
|
|
|
|
|
|
|
output = self.sendHeader()
|
|
|
|
|
2016-03-16 00:37:31 +01:00
|
|
|
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
|
|
|
|
yield "This function is disabled on this proxy"
|
|
|
|
raise StopIteration
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
@contextmanager
|
|
|
|
def benchmark(name, standard):
|
2019-03-16 02:27:04 +01:00
|
|
|
self.log.debug("Benchmark: %s" % name)
|
2015-07-12 20:36:46 +02:00
|
|
|
s = time.time()
|
2019-03-16 02:27:04 +01:00
|
|
|
output(b"- %s" % name.encode())
|
2015-07-12 20:36:46 +02:00
|
|
|
try:
|
|
|
|
yield 1
|
2019-03-16 02:27:04 +01:00
|
|
|
except Exception as err:
|
|
|
|
self.log.exception(err)
|
2019-03-23 03:40:42 +01:00
|
|
|
output(b"<br><b>! Error: %s</b><br>" % Debug.formatException(err).encode())
|
2015-07-12 20:36:46 +02:00
|
|
|
taken = time.time() - s
|
2017-01-22 21:22:53 +01:00
|
|
|
if taken > 0:
|
|
|
|
multipler = standard / taken
|
|
|
|
else:
|
|
|
|
multipler = 99
|
2015-07-12 20:36:46 +02:00
|
|
|
if multipler < 0.3:
|
|
|
|
speed = "Sloooow"
|
|
|
|
elif multipler < 0.5:
|
|
|
|
speed = "Ehh"
|
|
|
|
elif multipler < 0.8:
|
|
|
|
speed = "Goodish"
|
|
|
|
elif multipler < 1.2:
|
|
|
|
speed = "OK"
|
|
|
|
elif multipler < 1.7:
|
|
|
|
speed = "Fine"
|
|
|
|
elif multipler < 2.5:
|
|
|
|
speed = "Fast"
|
|
|
|
elif multipler < 3.5:
|
|
|
|
speed = "WOW"
|
|
|
|
else:
|
|
|
|
speed = "Insane!!"
|
2019-03-16 02:27:04 +01:00
|
|
|
output(b"%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed.encode()))
|
2015-07-12 20:36:46 +02:00
|
|
|
time.sleep(0.01)
|
|
|
|
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace }
|
|
|
|
table * { text-align: right; padding: 0px 10px }
|
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
|
|
|
yield "Benchmarking ZeroNet %s (rev%s) Python %s on: %s...<br>" % (config.version, config.rev, sys.version, sys.platform)
|
|
|
|
|
|
|
|
t = time.time()
|
|
|
|
|
2015-09-13 23:17:13 +02:00
|
|
|
# CryptBitcoin
|
2015-07-12 20:36:46 +02:00
|
|
|
yield "<br>CryptBitcoin:<br>"
|
|
|
|
from Crypt import CryptBitcoin
|
|
|
|
|
|
|
|
# seed = CryptBitcoin.newSeed()
|
|
|
|
# yield "- Seed: %s<br>" % seed
|
|
|
|
seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38"
|
|
|
|
|
|
|
|
with benchmark("hdPrivatekey x 10", 0.7):
|
|
|
|
for i in range(10):
|
|
|
|
privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10)
|
|
|
|
yield "."
|
|
|
|
valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk"
|
|
|
|
assert privatekey == valid, "%s != %s" % (privatekey, valid)
|
|
|
|
|
|
|
|
data = "Hello" * 1024 # 5k
|
|
|
|
with benchmark("sign x 10", 0.35):
|
|
|
|
for i in range(10):
|
|
|
|
yield "."
|
|
|
|
sign = CryptBitcoin.sign(data, privatekey)
|
2017-04-06 23:46:32 +02:00
|
|
|
valid = "G1GXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOiBHB+kp4cRPZOL7l1yqK5BHa6J+W97bMjvTXtxzljp6w="
|
2015-07-12 20:36:46 +02:00
|
|
|
assert sign == valid, "%s != %s" % (sign, valid)
|
|
|
|
|
|
|
|
address = CryptBitcoin.privatekeyToAddress(privatekey)
|
2019-03-16 02:27:04 +01:00
|
|
|
for lib_verify in ["btctools", "openssl", "libsecp256k1"]:
|
|
|
|
try:
|
|
|
|
CryptBitcoin.loadLib(lib_verify)
|
|
|
|
loaded = True
|
|
|
|
except Exception as err:
|
|
|
|
yield "- Error loading %s: %s" % (lib_verify, err)
|
|
|
|
loaded = False
|
|
|
|
if not loaded:
|
|
|
|
continue
|
|
|
|
with benchmark("%s verify x 100" % lib_verify, 0.37):
|
2015-07-12 20:36:46 +02:00
|
|
|
for i in range(100):
|
|
|
|
if i % 10 == 0:
|
|
|
|
yield "."
|
2019-03-16 02:27:04 +01:00
|
|
|
ok = CryptBitcoin.verify(data, address, sign, lib_verify=lib_verify)
|
2015-07-12 20:36:46 +02:00
|
|
|
assert ok, "does not verify from %s" % address
|
|
|
|
|
2015-09-13 23:17:13 +02:00
|
|
|
# CryptHash
|
2015-07-12 20:36:46 +02:00
|
|
|
yield "<br>CryptHash:<br>"
|
|
|
|
from Crypt import CryptHash
|
2019-03-16 02:27:04 +01:00
|
|
|
import io
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
data = io.BytesIO(b"Hello" * 1024 * 1024) # 5m
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
with benchmark("sha256 5M x 10", 0.6):
|
2015-07-12 20:36:46 +02:00
|
|
|
for i in range(10):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
data.seek(0)
|
|
|
|
hash = CryptHash.sha256sum(data)
|
|
|
|
yield "."
|
|
|
|
valid = "8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa"
|
|
|
|
assert hash == valid, "%s != %s" % (hash, valid)
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
data = io.BytesIO(b"Hello" * 1024 * 1024) # 5m
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
with benchmark("sha512 5M x 10", 0.6):
|
|
|
|
for i in range(10):
|
|
|
|
data.seek(0)
|
|
|
|
hash = CryptHash.sha512sum(data)
|
2015-07-12 20:36:46 +02:00
|
|
|
yield "."
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
valid = "9ca7e855d430964d5b55b114e95c6bbb114a6d478f6485df93044d87b108904d"
|
2015-07-12 20:36:46 +02:00
|
|
|
assert hash == valid, "%s != %s" % (hash, valid)
|
|
|
|
|
2017-02-11 18:26:21 +01:00
|
|
|
with benchmark("os.urandom(256) x 1000", 0.0065):
|
2015-09-13 23:17:13 +02:00
|
|
|
for i in range(10):
|
2017-02-11 18:26:21 +01:00
|
|
|
for y in range(100):
|
2015-09-13 23:17:13 +02:00
|
|
|
data = os.urandom(256)
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
# Msgpack
|
2019-03-16 02:27:04 +01:00
|
|
|
from util import Msgpack
|
|
|
|
yield "<br>Msgpack: (version: %s)<br>" % ".".join(map(str, Msgpack.msgpack.version))
|
|
|
|
binary = b'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv'
|
2019-03-23 03:40:42 +01:00
|
|
|
data = OrderedDict(
|
|
|
|
sorted({"int": 1024 * 1024 * 1024, "float": 12345.67890, "text": "hello" * 1024, "binary": binary}.items())
|
|
|
|
)
|
|
|
|
data_packed_valid = b'\x84\xa6binary\xc5\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00'
|
|
|
|
data_packed_valid += b"hello" * 1024
|
2015-09-13 23:17:13 +02:00
|
|
|
with benchmark("pack 5K x 10 000", 0.78):
|
|
|
|
for i in range(10):
|
|
|
|
for y in range(1000):
|
2019-03-16 02:27:04 +01:00
|
|
|
data_packed = Msgpack.pack(data)
|
2015-09-13 23:17:13 +02:00
|
|
|
yield "."
|
2019-03-23 03:40:42 +01:00
|
|
|
assert data_packed == data_packed_valid, "%s<br>!=<br>%s" % (repr(data_packed), repr(data_packed_valid))
|
2015-09-13 23:17:13 +02:00
|
|
|
|
|
|
|
with benchmark("unpack 5K x 10 000", 1.2):
|
|
|
|
for i in range(10):
|
|
|
|
for y in range(1000):
|
2019-03-16 02:27:04 +01:00
|
|
|
data_unpacked = Msgpack.unpack(data_packed, decode=False)
|
2015-09-13 23:17:13 +02:00
|
|
|
yield "."
|
2017-07-31 13:08:22 +02:00
|
|
|
assert data == data_unpacked, "%s != %s" % (data_unpacked, data)
|
2015-09-13 23:17:13 +02:00
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
for fallback in [True, False]:
|
|
|
|
with benchmark("streaming unpack 5K x 10 000 (fallback: %s)" % fallback, 1.4):
|
|
|
|
for i in range(10):
|
|
|
|
unpacker = Msgpack.getUnpacker(decode=False, fallback=fallback)
|
|
|
|
for y in range(1000):
|
|
|
|
unpacker.feed(data_packed)
|
|
|
|
for data_unpacked in unpacker:
|
|
|
|
pass
|
|
|
|
yield "."
|
|
|
|
assert data == data_unpacked, "%s != %s" % (data_unpacked, data)
|
2015-09-13 23:17:13 +02:00
|
|
|
|
|
|
|
# Db
|
2015-07-12 20:36:46 +02:00
|
|
|
from Db import Db
|
2017-02-11 18:25:55 +01:00
|
|
|
import sqlite3
|
|
|
|
yield "<br>Db: (version: %s, API: %s)<br>" % (sqlite3.sqlite_version, sqlite3.version)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
schema = {
|
|
|
|
"db_name": "TestDb",
|
|
|
|
"db_file": "%s/benchmark.db" % config.data_dir,
|
|
|
|
"maps": {
|
|
|
|
".*": {
|
|
|
|
"to_table": {
|
|
|
|
"test": "test"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"tables": {
|
|
|
|
"test": {
|
|
|
|
"cols": [
|
|
|
|
["test_id", "INTEGER"],
|
|
|
|
["title", "TEXT"],
|
|
|
|
["json_id", "INTEGER REFERENCES json (json_id)"]
|
|
|
|
],
|
|
|
|
"indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"],
|
|
|
|
"schema_changed": 1426195822
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if os.path.isfile("%s/benchmark.db" % config.data_dir):
|
|
|
|
os.unlink("%s/benchmark.db" % config.data_dir)
|
|
|
|
|
|
|
|
with benchmark("Open x 10", 0.13):
|
|
|
|
for i in range(10):
|
|
|
|
db = Db(schema, "%s/benchmark.db" % config.data_dir)
|
|
|
|
db.checkTables()
|
|
|
|
db.close()
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
db = Db(schema, "%s/benchmark.db" % config.data_dir)
|
|
|
|
db.checkTables()
|
|
|
|
import json
|
|
|
|
|
|
|
|
with benchmark("Insert x 10 x 1000", 1.0):
|
|
|
|
for u in range(10): # 10 user
|
|
|
|
data = {"test": []}
|
|
|
|
for i in range(1000): # 1000 line of data
|
|
|
|
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
|
|
|
|
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
|
2017-02-09 01:57:59 +01:00
|
|
|
db.updateJson("%s/test_%s.json" % (config.data_dir, u))
|
2015-07-12 20:36:46 +02:00
|
|
|
os.unlink("%s/test_%s.json" % (config.data_dir, u))
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
with benchmark("Buffered insert x 100 x 100", 1.3):
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.logging = False
|
|
|
|
for u in range(100, 200): # 100 user
|
|
|
|
data = {"test": []}
|
|
|
|
for i in range(100): # 1000 line of data
|
|
|
|
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
|
|
|
|
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
|
2017-02-09 01:57:59 +01:00
|
|
|
db.updateJson("%s/test_%s.json" % (config.data_dir, u), cur=cur)
|
2015-07-12 20:36:46 +02:00
|
|
|
os.unlink("%s/test_%s.json" % (config.data_dir, u))
|
|
|
|
if u % 10 == 0:
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
yield " - Total rows in db: %s<br>" % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0]
|
|
|
|
|
|
|
|
with benchmark("Indexed query x 1000", 0.25):
|
|
|
|
found = 0
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.logging = False
|
|
|
|
for i in range(1000): # 1000x by test_id
|
|
|
|
res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i)
|
|
|
|
for row in res:
|
|
|
|
found += 1
|
|
|
|
if i % 100 == 0:
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
assert found == 20000, "Found: %s != 20000" % found
|
|
|
|
|
|
|
|
with benchmark("Not indexed query x 100", 0.6):
|
|
|
|
found = 0
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.logging = False
|
|
|
|
for i in range(100): # 1000x by test_id
|
|
|
|
res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i)
|
|
|
|
for row in res:
|
|
|
|
found += 1
|
|
|
|
if i % 10 == 0:
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
assert found == 18900, "Found: %s != 18900" % found
|
|
|
|
|
|
|
|
with benchmark("Like query x 100", 1.8):
|
|
|
|
found = 0
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.logging = False
|
|
|
|
for i in range(100): # 1000x by test_id
|
|
|
|
res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i)
|
|
|
|
for row in res:
|
|
|
|
found += 1
|
|
|
|
if i % 10 == 0:
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
assert found == 38900, "Found: %s != 11000" % found
|
|
|
|
|
|
|
|
db.close()
|
|
|
|
if os.path.isfile("%s/benchmark.db" % config.data_dir):
|
|
|
|
os.unlink("%s/benchmark.db" % config.data_dir)
|
|
|
|
|
|
|
|
gc.collect() # Implicit grabage collection
|
|
|
|
|
2017-04-13 22:54:16 +02:00
|
|
|
# Zip
|
|
|
|
yield "<br>Compression:<br>"
|
|
|
|
import zipfile
|
2019-03-16 02:27:04 +01:00
|
|
|
test_data = b"Test" * 1024
|
|
|
|
file_name = b"\xc3\x81rv\xc3\xadzt\xc5\xb0r\xc5\x91t\xc3\xbck\xc3\xb6r\xc3\xb3g\xc3\xa9p\xe4\xb8\xad\xe5\x8d\x8e%s.txt".decode("utf8")
|
2017-04-13 22:54:16 +02:00
|
|
|
|
2017-04-14 11:52:01 +02:00
|
|
|
with benchmark("Zip pack x 10", 0.12):
|
2017-04-13 22:54:16 +02:00
|
|
|
for i in range(10):
|
|
|
|
with zipfile.ZipFile('%s/test.zip' % config.data_dir, 'w') as archive:
|
|
|
|
for y in range(100):
|
2017-04-14 11:52:01 +02:00
|
|
|
zip_info = zipfile.ZipInfo(file_name % y, (1980,1,1,0,0,0))
|
|
|
|
zip_info.compress_type = zipfile.ZIP_DEFLATED
|
|
|
|
zip_info.create_system = 3
|
2019-03-16 02:27:04 +01:00
|
|
|
zip_info.flag_bits = 0
|
2019-03-23 03:40:42 +01:00
|
|
|
zip_info.external_attr = 25165824
|
2017-04-14 11:52:01 +02:00
|
|
|
archive.writestr(zip_info, test_data)
|
2017-04-13 22:54:16 +02:00
|
|
|
yield "."
|
|
|
|
|
2017-04-14 11:52:01 +02:00
|
|
|
hash = CryptHash.sha512sum(open("%s/test.zip" % config.data_dir, "rb"))
|
2019-03-16 02:27:04 +01:00
|
|
|
valid = "f630fece29fff1cc8dbf454e47a87fea2746a4dbbd2ceec098afebab45301562"
|
2017-04-14 11:52:01 +02:00
|
|
|
assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid)
|
|
|
|
|
|
|
|
with benchmark("Zip unpack x 10", 0.2):
|
2017-04-13 22:54:16 +02:00
|
|
|
for i in range(10):
|
|
|
|
with zipfile.ZipFile('%s/test.zip' % config.data_dir) as archive:
|
|
|
|
for y in range(100):
|
2019-03-16 02:27:04 +01:00
|
|
|
data = archive.open(file_name % y).read()
|
|
|
|
assert archive.open(file_name % y).read() == test_data, "Invalid data: %s..." % data[0:30]
|
2017-04-13 22:54:16 +02:00
|
|
|
yield "."
|
|
|
|
|
|
|
|
if os.path.isfile("%s/test.zip" % config.data_dir):
|
|
|
|
os.unlink("%s/test.zip" % config.data_dir)
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
# gz, bz2, xz
|
2017-04-13 22:54:16 +02:00
|
|
|
import tarfile
|
2019-03-16 02:27:04 +01:00
|
|
|
import gzip
|
2017-04-13 22:54:16 +02:00
|
|
|
|
|
|
|
# Monkey patch _init_write_gz to use fixed date in order to keep the hash independent from datetime
|
|
|
|
def nodate_write_gzip_header(self):
|
2019-03-16 02:27:04 +01:00
|
|
|
self._write_mtime = 0
|
2017-04-13 22:54:16 +02:00
|
|
|
original_write_gzip_header(self)
|
|
|
|
|
|
|
|
original_write_gzip_header = gzip.GzipFile._write_gzip_header
|
|
|
|
gzip.GzipFile._write_gzip_header = nodate_write_gzip_header
|
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
test_data_io = io.BytesIO(b"Test" * 1024)
|
|
|
|
archive_formats = {
|
|
|
|
"gz": {"hash": "4704ebd8c987ed6f833059f1de9c475d443b0539b8d4c4cb8b49b26f7bbf2d19", "time_pack": 0.3, "time_unpack": 0.2},
|
|
|
|
"bz2": {"hash": "90cba0b4d9abaa37b830bf37e4adba93bfd183e095b489ebee62aaa94339f3b5", "time_pack": 2.0, "time_unpack": 0.5},
|
|
|
|
"xz": {"hash": "37abc16d552cfd4a495cb2acbf8b1d5877631d084f6571f4d6544bc548c69bae", "time_pack": 1.4, "time_unpack": 0.2}
|
|
|
|
}
|
|
|
|
for ext, format_data in archive_formats.items():
|
|
|
|
archive_path = '%s/test.tar.%s' % (config.data_dir, ext)
|
|
|
|
with benchmark("Tar.%s pack x 10" % ext, format_data["time_pack"]):
|
|
|
|
for i in range(10):
|
|
|
|
with tarfile.open(archive_path, 'w:%s' % ext) as archive:
|
|
|
|
for y in range(100):
|
|
|
|
test_data_io.seek(0)
|
|
|
|
tar_info = tarfile.TarInfo(file_name % y)
|
|
|
|
tar_info.size = 4 * 1024
|
|
|
|
archive.addfile(tar_info, test_data_io)
|
|
|
|
yield "."
|
2017-04-14 11:52:01 +02:00
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
hash = CryptHash.sha512sum(open("%s/test.tar.%s" % (config.data_dir, ext), "rb"))
|
|
|
|
valid = format_data["hash"]
|
|
|
|
assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid)
|
2017-04-13 22:54:16 +02:00
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
archive_size = os.path.getsize(archive_path) / 1024
|
|
|
|
with benchmark("Tar.%s unpack (%.2fkB) x 10" % (ext, archive_size), format_data["time_unpack"]):
|
|
|
|
for i in range(10):
|
|
|
|
with tarfile.open(archive_path, 'r:%s' % ext) as archive:
|
|
|
|
for y in range(100):
|
|
|
|
assert archive.extractfile(file_name % y).read() == test_data
|
|
|
|
yield "."
|
2017-04-13 22:54:16 +02:00
|
|
|
|
2019-03-16 02:27:04 +01:00
|
|
|
if os.path.isfile(archive_path):
|
|
|
|
os.unlink(archive_path)
|
2017-04-13 22:54:16 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
yield "<br>Done. Total: %.2fs" % (time.time() - t)
|
|
|
|
|
|
|
|
def actionGcCollect(self):
|
|
|
|
import gc
|
|
|
|
self.sendHeader()
|
|
|
|
yield str(gc.collect())
|