version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
import re, time, cgi, os
|
|
|
|
from Plugin import PluginManager
|
2015-04-12 23:59:22 +02:00
|
|
|
from Config import config
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
|
|
|
@PluginManager.registerTo("UiRequest")
|
|
|
|
class UiRequestPlugin(object):
|
|
|
|
def formatTableRow(self, row):
|
|
|
|
back = []
|
|
|
|
for format, val in row:
|
|
|
|
if val == None:
|
|
|
|
formatted = "n/a"
|
|
|
|
elif format == "since":
|
|
|
|
if val:
|
|
|
|
formatted = "%.0f" % (time.time()-val)
|
|
|
|
else:
|
|
|
|
formatted = "n/a"
|
|
|
|
else:
|
|
|
|
formatted = format % val
|
|
|
|
back.append("<td>%s</td>" % formatted)
|
|
|
|
return "<tr>%s</tr>" % "".join(back)
|
|
|
|
|
|
|
|
|
|
|
|
def getObjSize(self, obj, hpy = None):
|
|
|
|
if hpy:
|
|
|
|
return float(hpy.iso(obj).domisize)/1024
|
|
|
|
else:
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
# /Stats entry point
|
|
|
|
def actionStats(self):
|
|
|
|
import gc, sys
|
|
|
|
from Ui import UiRequest
|
|
|
|
|
|
|
|
hpy = None
|
|
|
|
if self.get.get("size") == "1": # Calc obj size
|
|
|
|
try:
|
|
|
|
import guppy
|
|
|
|
hpy = guppy.hpy()
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
self.sendHeader()
|
|
|
|
s = time.time()
|
|
|
|
main = sys.modules["main"]
|
|
|
|
|
|
|
|
# Style
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace }
|
|
|
|
table * { text-align: right; padding: 0px 10px }
|
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
|
|
|
# Memory
|
|
|
|
try:
|
2015-04-20 02:56:33 +02:00
|
|
|
yield "rev%s | " % config.rev
|
2015-04-15 02:54:10 +02:00
|
|
|
yield "IP external: %s | " % config.ip_external
|
|
|
|
yield "Opened: %s | " % main.file_server.port_opened
|
|
|
|
yield "Recv: %.2fMB, Sent: %.2fMB | " % (float(main.file_server.bytes_recv)/1024/1024, float(main.file_server.bytes_sent)/1024/1024)
|
|
|
|
yield "Peerid: %s | " % main.file_server.peer_id
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
import psutil
|
|
|
|
process = psutil.Process(os.getpid())
|
|
|
|
mem = process.get_memory_info()[0] / float(2 ** 20)
|
2015-04-15 02:54:10 +02:00
|
|
|
yield "Mem: %.2fMB | " % mem
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
yield "Threads: %s | " % len(process.threads())
|
|
|
|
yield "CPU: usr %.2fs sys %.2fs | " % process.cpu_times()
|
|
|
|
yield "Open files: %s | " % len(process.open_files())
|
2015-04-12 23:59:22 +02:00
|
|
|
yield "Sockets: %s | " % len(process.connections())
|
|
|
|
yield "Calc size <a href='?size=1'>on</a> <a href='?size=0'>off</a>"
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
except Exception, err:
|
|
|
|
pass
|
2015-04-12 23:59:22 +02:00
|
|
|
yield "<br>"
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
2015-04-03 02:47:51 +02:00
|
|
|
# Connections
|
2015-04-15 02:54:10 +02:00
|
|
|
yield "<b>Connections</b> (%s, total made: %s):<br>" % (len(main.file_server.connections), main.file_server.last_connection_id)
|
2015-04-12 23:59:22 +02:00
|
|
|
yield "<table><tr> <th>id</th> <th>protocol</th> <th>type</th> <th>ip</th> <th>open</th> <th>ping</th> <th>buff</th>"
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
yield "<th>idle</th> <th>open</th> <th>delay</th> <th>sent</th> <th>received</th> <th>last sent</th> <th>waiting</th> <th>version</th> <th>peerid</th> </tr>"
|
|
|
|
for connection in main.file_server.connections:
|
|
|
|
yield self.formatTableRow([
|
|
|
|
("%3d", connection.id),
|
|
|
|
("%s", connection.protocol),
|
|
|
|
("%s", connection.type),
|
2015-04-12 23:59:22 +02:00
|
|
|
("%s:%s", (connection.ip, connection.port)),
|
|
|
|
("%s", connection.handshake.get("port_opened")),
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
("%6.3f", connection.last_ping_delay),
|
|
|
|
("%s", connection.incomplete_buff_recv),
|
|
|
|
("since", max(connection.last_send_time, connection.last_recv_time)),
|
|
|
|
("since", connection.start_time),
|
|
|
|
("%.3f", connection.last_sent_time-connection.last_send_time),
|
|
|
|
("%.0fkB", connection.bytes_sent/1024),
|
|
|
|
("%.0fkB", connection.bytes_recv/1024),
|
|
|
|
("%s", connection.last_cmd),
|
|
|
|
("%s", connection.waiting_requests.keys()),
|
2015-04-17 23:12:22 +02:00
|
|
|
("%s r%s", (connection.handshake.get("version"), connection.handshake.get("rev", "?")) ),
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
("%s", connection.handshake.get("peer_id")),
|
|
|
|
])
|
|
|
|
yield "</table>"
|
|
|
|
|
2015-04-03 02:47:51 +02:00
|
|
|
|
|
|
|
# Sites
|
|
|
|
yield "<br><br><b>Sites</b>:"
|
|
|
|
yield "<table>"
|
2015-04-13 23:08:57 +02:00
|
|
|
yield "<tr><th>address</th> <th>connected</th> <th>peers</th> <th>content.json</th> </tr>"
|
2015-04-03 02:47:51 +02:00
|
|
|
for site in self.server.sites.values():
|
|
|
|
yield self.formatTableRow([
|
2015-04-15 02:54:10 +02:00
|
|
|
("<a href='#ShowPeers' onclick='document.getElementById(\"peers_%s\").style.display=\"initial\"; return false'>%s</a>", (site.address, site.address)),
|
2015-04-03 02:47:51 +02:00
|
|
|
("%s", [peer.connection.id for peer in site.peers.values() if peer.connection and peer.connection.connected]),
|
2015-04-17 00:34:08 +02:00
|
|
|
("%s/%s/%s", (
|
|
|
|
len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected]),
|
|
|
|
len(site.getConnectablePeers(100)),
|
|
|
|
len(site.peers)
|
|
|
|
) ),
|
2015-04-03 02:47:51 +02:00
|
|
|
("%s", len(site.content_manager.contents)),
|
|
|
|
])
|
2015-04-15 23:33:21 +02:00
|
|
|
yield "<tr><td id='peers_%s' style='display: none; white-space: pre'>" % site.address
|
2015-04-15 02:54:10 +02:00
|
|
|
for key, peer in site.peers.items():
|
2015-04-15 23:33:21 +02:00
|
|
|
yield "(%s, err: %s) %22s -<br>" % (peer.connection, peer.connection_error, key)
|
2015-04-15 02:54:10 +02:00
|
|
|
yield "<br></td></tr>"
|
2015-04-03 02:47:51 +02:00
|
|
|
yield "</table>"
|
|
|
|
|
|
|
|
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
# Object types
|
2015-04-08 01:57:55 +02:00
|
|
|
|
|
|
|
obj_count = {}
|
|
|
|
for obj in gc.get_objects():
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if not obj_type in obj_count:
|
|
|
|
obj_count[obj_type] = [0, 0]
|
|
|
|
obj_count[obj_type][0] += 1 # Count
|
|
|
|
obj_count[obj_type][1] += float(sys.getsizeof(obj))/1024 # Size
|
|
|
|
|
2015-04-20 22:31:29 +02:00
|
|
|
yield "<br><br><b>Objects in memory (types: %s, total: %s, %.2fkb):</b><br>" % (len(obj_count), sum([stat[0] for stat in obj_count.values()]), sum([stat[1] for stat in obj_count.values()]))
|
2015-04-08 01:57:55 +02:00
|
|
|
|
|
|
|
for obj, stat in sorted(obj_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count
|
2015-04-19 15:38:41 +02:00
|
|
|
yield " - %.1fkb = %s x <a href=\"/Listobj?type=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))
|
2015-04-08 01:57:55 +02:00
|
|
|
|
2015-04-03 02:47:51 +02:00
|
|
|
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
# Classes
|
|
|
|
|
|
|
|
class_count = {}
|
|
|
|
for obj in gc.get_objects():
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if obj_type != "<type 'instance'>": continue
|
|
|
|
class_name = obj.__class__.__name__
|
|
|
|
if not class_name in class_count:
|
|
|
|
class_count[class_name] = [0, 0]
|
|
|
|
class_count[class_name][0] += 1 # Count
|
|
|
|
class_count[class_name][1] += float(sys.getsizeof(obj))/1024 # Size
|
|
|
|
|
|
|
|
yield "<br><br><b>Classes in memory (types: %s, total: %s, %.2fkb):</b><br>" % (len(class_count), sum([stat[0] for stat in class_count.values()]), sum([stat[1] for stat in class_count.values()]))
|
|
|
|
|
|
|
|
for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count
|
|
|
|
yield " - %.1fkb = %s x <a href=\"/Dumpobj?class=%s\">%s</a><br>" % (stat[1], stat[0], obj, cgi.escape(obj))
|
|
|
|
|
|
|
|
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
from greenlet import greenlet
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
|
|
|
|
yield "<br>Greenlets (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2015-04-08 01:57:55 +02:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
|
|
|
|
|
|
|
from Worker import Worker
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
|
|
|
|
yield "<br>Workers (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2015-04-08 01:57:55 +02:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
|
|
|
|
|
|
|
from Connection import Connection
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
|
|
|
|
yield "<br>Connections (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2015-04-08 01:57:55 +02:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
2015-04-03 02:47:51 +02:00
|
|
|
|
2015-04-28 00:43:17 +02:00
|
|
|
from socket import socket
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
|
|
|
|
yield "<br>Sockets (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
2015-04-03 02:47:51 +02:00
|
|
|
|
|
|
|
from msgpack import Unpacker
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
|
|
|
|
yield "<br>Msgpack unpacker (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2015-04-08 01:57:55 +02:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
|
|
|
|
|
|
|
from Site import Site
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
|
|
|
|
yield "<br>Sites (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2015-04-08 01:57:55 +02:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
|
|
|
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
|
|
|
|
yield "<br>Loggers (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
2015-04-08 01:57:55 +02:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj.name)))
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
|
|
|
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
|
2015-04-20 22:31:29 +02:00
|
|
|
yield "<br>UiRequests (%s):<br>" % len(objs)
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
for obj in objs:
|
2015-04-08 01:57:55 +02:00
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
|
2015-04-20 22:31:29 +02:00
|
|
|
|
|
|
|
from Peer import Peer
|
|
|
|
objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
|
|
|
|
yield "<br>Peers (%s):<br>" % len(objs)
|
|
|
|
for obj in objs:
|
|
|
|
yield " - %.1fkb: %s<br>" % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
|
|
|
|
|
|
|
|
|
version 0.2.7, plugin system, multiuser plugin for zeroproxies, reworked imports, cookie parse, stats moved to plugin, usermanager class, dont generate site auth on listing, multiline notifications, allow server side prompt from user, update script keep plugins disabled status
2015-03-24 01:33:09 +01:00
|
|
|
objs = [(key, val) for key, val in sys.modules.iteritems() if val is not None]
|
|
|
|
objs.sort()
|
|
|
|
yield "<br>Modules (%s):<br>" % len(objs)
|
|
|
|
for module_name, module in objs:
|
|
|
|
yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module)))
|
|
|
|
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
gc.collect() # Implicit grabage collection
|
2015-04-08 01:57:55 +02:00
|
|
|
yield "Done in %.1f" % (time.time()-s)
|
2015-04-18 03:02:08 +02:00
|
|
|
|
|
|
|
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
def actionDumpobj(self):
|
|
|
|
import gc, sys
|
|
|
|
|
|
|
|
self.sendHeader()
|
|
|
|
class_filter = self.get.get("class")
|
|
|
|
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace; white-space: pre }
|
|
|
|
table * { text-align: right; padding: 0px 10px }
|
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
|
|
|
objs = gc.get_objects()
|
|
|
|
for obj in objs:
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if obj_type != "<type 'instance'>" or obj.__class__.__name__ != class_filter: continue
|
|
|
|
yield "%.1fkb %s... " % (float(sys.getsizeof(obj))/1024, cgi.escape(str(obj)) )
|
|
|
|
for attr in dir(obj):
|
|
|
|
yield "- %s: %s<br>" % (attr, cgi.escape(str(getattr(obj, attr))))
|
|
|
|
yield "<br>"
|
|
|
|
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
gc.collect() # Implicit grabage collection
|
|
|
|
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
|
2015-04-19 15:38:41 +02:00
|
|
|
def actionListobj(self):
|
|
|
|
import gc, sys
|
|
|
|
|
|
|
|
self.sendHeader()
|
|
|
|
type_filter = self.get.get("type")
|
|
|
|
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace; white-space: pre }
|
|
|
|
table * { text-align: right; padding: 0px 10px }
|
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
|
|
|
yield "Listing all %s objects in memory...<br>" % cgi.escape(type_filter)
|
|
|
|
|
|
|
|
ref_count = {}
|
|
|
|
objs = gc.get_objects()
|
|
|
|
for obj in objs:
|
|
|
|
obj_type = str(type(obj))
|
|
|
|
if obj_type != type_filter: continue
|
|
|
|
refs = [ref for ref in gc.get_referrers(obj) if hasattr(ref, "__class__") and ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"]]
|
|
|
|
if not refs: continue
|
2015-04-20 22:31:29 +02:00
|
|
|
yield "%.1fkb <span title=\"%s\">%s</span>... " % (float(sys.getsizeof(obj))/1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100)) )
|
2015-04-19 15:38:41 +02:00
|
|
|
for ref in refs:
|
|
|
|
yield " ["
|
|
|
|
if "object at" in str(ref) or len(str(ref)) > 100:
|
|
|
|
yield str(ref.__class__.__name__)
|
|
|
|
else:
|
|
|
|
yield str(ref.__class__.__name__)+":"+cgi.escape(str(ref))
|
|
|
|
yield "] "
|
|
|
|
ref_type = ref.__class__.__name__
|
|
|
|
if ref_type not in ref_count:
|
|
|
|
ref_count[ref_type] = [0,0]
|
|
|
|
ref_count[ref_type][0] += 1 # Count
|
|
|
|
ref_count[ref_type][1] += float(sys.getsizeof(obj))/1024 # Size
|
|
|
|
yield "<br>"
|
|
|
|
|
|
|
|
yield "<br>Object referrer (total: %s, %.2fkb):<br>" % (len(ref_count), sum([stat[1] for stat in ref_count.values()]))
|
|
|
|
|
|
|
|
for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count
|
|
|
|
yield " - %.1fkb = %s x %s<br>" % (stat[1], stat[0], cgi.escape(str(obj)) )
|
|
|
|
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
gc.collect() # Implicit grabage collection
|
|
|
|
|
2015-04-19 15:38:41 +02:00
|
|
|
|
2015-04-18 03:02:08 +02:00
|
|
|
def actionBenchmark(self):
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
import sys, gc
|
2015-04-18 03:02:08 +02:00
|
|
|
from contextlib import contextmanager
|
|
|
|
|
|
|
|
output = self.sendHeader()
|
|
|
|
|
|
|
|
@contextmanager
|
|
|
|
def benchmark(name, standard):
|
|
|
|
s = time.time()
|
|
|
|
output("- %s" % name)
|
|
|
|
try:
|
|
|
|
yield 1
|
|
|
|
except Exception, err:
|
|
|
|
output("<br><b>! Error: %s</b><br>" % err)
|
|
|
|
taken = time.time()-s
|
|
|
|
multipler = standard/taken
|
|
|
|
if multipler < 0.3: speed = "Sloooow"
|
|
|
|
elif multipler < 0.5: speed = "Ehh"
|
|
|
|
elif multipler < 0.8: speed = "Goodish"
|
|
|
|
elif multipler < 1.2: speed = "OK"
|
|
|
|
elif multipler < 1.7: speed = "Fine"
|
|
|
|
elif multipler < 2.5: speed = "Fast"
|
|
|
|
elif multipler < 3.5: speed = "WOW"
|
|
|
|
else: speed = "Insane!!"
|
|
|
|
output("%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed))
|
|
|
|
time.sleep(0.01)
|
|
|
|
|
|
|
|
|
|
|
|
yield """
|
|
|
|
<style>
|
|
|
|
* { font-family: monospace }
|
|
|
|
table * { text-align: right; padding: 0px 10px }
|
|
|
|
</style>
|
|
|
|
"""
|
|
|
|
|
|
|
|
yield "Benchmarking ZeroNet %s (rev%s) Python %s, platform: %s...<br>" % (config.version, config.rev, sys.version, sys.platform)
|
|
|
|
|
|
|
|
t = time.time()
|
|
|
|
|
|
|
|
yield "<br>CryptBitcoin:<br>"
|
|
|
|
from Crypt import CryptBitcoin
|
|
|
|
|
|
|
|
# seed = CryptBitcoin.newSeed()
|
|
|
|
# yield "- Seed: %s<br>" % seed
|
|
|
|
seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38"
|
|
|
|
|
|
|
|
with benchmark("hdPrivatekey x 10", 0.7):
|
|
|
|
for i in range(10):
|
|
|
|
privatekey = CryptBitcoin.hdPrivatekey(seed, i*10)
|
|
|
|
yield "."
|
|
|
|
valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk"
|
|
|
|
assert privatekey == valid, "%s != %s" % (privatekey, valid)
|
|
|
|
|
|
|
|
|
|
|
|
data = "Hello"*1024 #5k
|
|
|
|
with benchmark("sign x 10", 0.35):
|
|
|
|
for i in range(10):
|
|
|
|
yield "."
|
|
|
|
sign = CryptBitcoin.sign(data, privatekey)
|
|
|
|
valid = "HFGXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOi+4+BbWHjuwmx0EaKNV1G+kP0tQDxWu0YApxwxZbSmZU="
|
|
|
|
assert sign == valid, "%s != %s" % (sign, valid)
|
|
|
|
|
|
|
|
|
|
|
|
address = CryptBitcoin.privatekeyToAddress(privatekey)
|
2015-04-28 00:43:17 +02:00
|
|
|
if CryptBitcoin.opensslVerify: # Openssl avalible
|
|
|
|
with benchmark("openssl verify x 100", 0.37):
|
|
|
|
for i in range(100):
|
|
|
|
if i%10==0: yield "."
|
|
|
|
ok = CryptBitcoin.verify(data, address, sign)
|
|
|
|
assert ok, "does not verify from %s" % address
|
|
|
|
else:
|
|
|
|
yield " - openssl verify x 100...not avalible :(<br>"
|
|
|
|
|
|
|
|
opensslVerify_bk = CryptBitcoin.opensslVerify # Emulate openssl not found in any way
|
|
|
|
CryptBitcoin.opensslVerify = None
|
|
|
|
with benchmark("pure-python verify x 10", 1.6):
|
2015-04-18 03:02:08 +02:00
|
|
|
for i in range(10):
|
|
|
|
yield "."
|
|
|
|
ok = CryptBitcoin.verify(data, address, sign)
|
|
|
|
assert ok, "does not verify from %s" % address
|
2015-04-28 00:43:17 +02:00
|
|
|
CryptBitcoin.opensslVerify = opensslVerify_bk
|
2015-04-18 03:02:08 +02:00
|
|
|
|
|
|
|
|
|
|
|
yield "<br>CryptHash:<br>"
|
|
|
|
from Crypt import CryptHash
|
|
|
|
from cStringIO import StringIO
|
|
|
|
|
|
|
|
data = StringIO("Hello"*1024*1024) #5m
|
|
|
|
with benchmark("sha512 x 10 000", 1):
|
|
|
|
for i in range(10):
|
|
|
|
for y in range(10000):
|
|
|
|
hash = CryptHash.sha512sum(data)
|
|
|
|
yield "."
|
|
|
|
valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce"
|
|
|
|
assert hash == valid, "%s != %s" % (hash, valid)
|
|
|
|
|
|
|
|
|
|
|
|
yield "<br>Db:<br>"
|
|
|
|
from Db import Db
|
|
|
|
|
|
|
|
schema = {
|
|
|
|
"db_name": "TestDb",
|
2015-05-31 15:52:21 +02:00
|
|
|
"db_file": "%s/benchmark.db" % config.data_dir,
|
2015-04-18 03:02:08 +02:00
|
|
|
"maps": {
|
|
|
|
".*": {
|
|
|
|
"to_table": {
|
|
|
|
"test": "test"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
},
|
|
|
|
"tables": {
|
|
|
|
"test": {
|
|
|
|
"cols": [
|
|
|
|
["test_id", "INTEGER"],
|
|
|
|
["title", "TEXT"],
|
|
|
|
["json_id", "INTEGER REFERENCES json (json_id)"]
|
|
|
|
],
|
|
|
|
"indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"],
|
|
|
|
"schema_changed": 1426195822
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-05-31 15:52:21 +02:00
|
|
|
if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir)
|
2015-04-18 03:02:08 +02:00
|
|
|
|
|
|
|
with benchmark("Open x 10", 0.13):
|
|
|
|
for i in range(10):
|
2015-05-31 15:52:21 +02:00
|
|
|
db = Db(schema, "%s/benchmark.db" % config.data_dir)
|
2015-04-18 03:02:08 +02:00
|
|
|
db.checkTables()
|
|
|
|
db.close()
|
|
|
|
yield "."
|
|
|
|
|
|
|
|
|
2015-05-31 15:52:21 +02:00
|
|
|
db = Db(schema, "%s/benchmark.db" % config.data_dir)
|
2015-04-18 03:02:08 +02:00
|
|
|
db.checkTables()
|
|
|
|
import json
|
|
|
|
|
|
|
|
with benchmark("Insert x 10 x 1000", 1.0):
|
|
|
|
for u in range(10): # 10 user
|
|
|
|
data = {"test": []}
|
|
|
|
for i in range(1000): # 1000 line of data
|
|
|
|
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
|
2015-05-31 15:52:21 +02:00
|
|
|
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
|
|
|
|
db.loadJson("%s/test_%s.json" % (config.data_dir, u))
|
|
|
|
os.unlink("%s/test_%s.json" % (config.data_dir, u))
|
2015-04-18 03:02:08 +02:00
|
|
|
yield "."
|
|
|
|
|
|
|
|
|
|
|
|
with benchmark("Buffered insert x 100 x 100", 1.3):
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.execute("BEGIN")
|
|
|
|
cur.logging = False
|
|
|
|
for u in range(100, 200): # 100 user
|
|
|
|
data = {"test": []}
|
|
|
|
for i in range(100): # 1000 line of data
|
|
|
|
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
|
2015-05-31 15:52:21 +02:00
|
|
|
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
|
|
|
|
db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur)
|
|
|
|
os.unlink("%s/test_%s.json" % (config.data_dir, u))
|
2015-04-18 03:02:08 +02:00
|
|
|
if u%10 == 0: yield "."
|
|
|
|
cur.execute("COMMIT")
|
|
|
|
|
|
|
|
yield " - Total rows in db: %s<br>" % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0]
|
|
|
|
|
|
|
|
with benchmark("Indexed query x 1000", 0.25):
|
|
|
|
found = 0
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.logging = False
|
|
|
|
for i in range(1000): # 1000x by test_id
|
|
|
|
res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i)
|
|
|
|
for row in res:
|
|
|
|
found += 1
|
|
|
|
if i%100 == 0: yield "."
|
|
|
|
|
|
|
|
assert found == 20000, "Found: %s != 20000" % found
|
|
|
|
|
|
|
|
|
|
|
|
with benchmark("Not indexed query x 100", 0.6):
|
|
|
|
found = 0
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.logging = False
|
|
|
|
for i in range(100): # 1000x by test_id
|
|
|
|
res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i)
|
|
|
|
for row in res:
|
|
|
|
found += 1
|
|
|
|
if i%10 == 0: yield "."
|
|
|
|
|
|
|
|
assert found == 18900, "Found: %s != 18900" % found
|
|
|
|
|
|
|
|
|
|
|
|
with benchmark("Like query x 100", 1.8):
|
|
|
|
found = 0
|
|
|
|
cur = db.getCursor()
|
|
|
|
cur.logging = False
|
|
|
|
for i in range(100): # 1000x by test_id
|
|
|
|
res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i)
|
|
|
|
for row in res:
|
|
|
|
found += 1
|
|
|
|
if i%10 == 0: yield "."
|
|
|
|
|
|
|
|
assert found == 38900, "Found: %s != 11000" % found
|
|
|
|
|
|
|
|
|
|
|
|
db.close()
|
2015-05-31 15:52:21 +02:00
|
|
|
if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir)
|
2015-04-18 03:02:08 +02:00
|
|
|
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
gc.collect() # Implicit grabage collection
|
|
|
|
|
|
|
|
yield "<br>Done. Total: %.2fs" % (time.time()-t)
|
2015-04-18 03:02:08 +02:00
|
|
|
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
|
|
|
|
def actionGcCollect(self):
|
|
|
|
import gc
|
|
|
|
self.sendHeader()
|
|
|
|
yield str(gc.collect())
|