import time
import cgi
import os
from Plugin import PluginManager
from Config import config
@PluginManager.registerTo("UiRequest")
class UiRequestPlugin(object):
def formatTableRow(self, row, class_name=""):
back = []
for format, val in row:
if val is None:
formatted = "n/a"
elif format == "since":
if val:
formatted = "%.0f" % (time.time() - val)
else:
formatted = "n/a"
else:
formatted = format % val
back.append("
Tor hidden services (status: %s): " % main.file_server.tor_manager.status
for site_address, onion in main.file_server.tor_manager.site_onions.items():
yield "- %-34s: %s " % (site_address, onion)
# Db
yield "
Db: "
for db in sys.modules["Db.Db"].opened_dbs:
yield "- %.3fs: %s " % (time.time() - db.last_query_time, db.db_path.encode("utf8"))
# Sites
yield "
Sites:"
yield "
"
yield "
address
connected
peers
content.json
out
in
"
for site in sorted(self.server.sites.values(), lambda a, b: cmp(a.address,b.address)):
yield self.formatTableRow([
(
"""%s""",
(site.address, site.address)
),
("%s", [peer.connection.id for peer in site.peers.values() if peer.connection and peer.connection.connected]),
("%s/%s/%s", (
len([peer for peer in site.peers.values() if peer.connection and peer.connection.connected]),
len(site.getConnectablePeers(100)),
len(site.peers)
)),
("%s (loaded: %s)", (
len(site.content_manager.contents),
len([key for key, val in dict(site.content_manager.contents).iteritems() if val])
)),
("%.0fkB", site.settings.get("bytes_sent", 0) / 1024),
("%.0fkB", site.settings.get("bytes_recv", 0) / 1024),
], "serving-%s" % site.settings["serving"])
yield "
Big files: "
for site in self.server.sites.values():
if not site.settings.get("has_bigfile"):
continue
bigfiles = {}
yield """%s """ % (site.address, site.address)
for peer in site.peers.values():
if not peer.time_piecefields_updated:
continue
for sha512, piecefield in peer.piecefields.iteritems():
if sha512 not in bigfiles:
bigfiles[sha512] = []
bigfiles[sha512].append(peer)
yield "
"
yield ""
# No more if not in debug mode
if not config.debug:
raise StopIteration
# Object types
obj_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type not in obj_count:
obj_count[obj_type] = [0, 0]
obj_count[obj_type][0] += 1 # Count
obj_count[obj_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "
Objects in memory (types: %s, total: %s, %.2fkb): " % (
len(obj_count),
sum([stat[0] for stat in obj_count.values()]),
sum([stat[1] for stat in obj_count.values()])
)
for obj, stat in sorted(obj_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count
yield " - %.1fkb = %s x %s " % (stat[1], stat[0], obj, cgi.escape(obj))
# Classes
class_count = {}
for obj in gc.get_objects():
obj_type = str(type(obj))
if obj_type != "":
continue
class_name = obj.__class__.__name__
if class_name not in class_count:
class_count[class_name] = [0, 0]
class_count[class_name][0] += 1 # Count
class_count[class_name][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield "
Classes in memory (types: %s, total: %s, %.2fkb): " % (
len(class_count),
sum([stat[0] for stat in class_count.values()]),
sum([stat[1] for stat in class_count.values()])
)
for obj, stat in sorted(class_count.items(), key=lambda x: x[1][0], reverse=True): # Sorted by count
yield " - %.1fkb = %s x %s " % (stat[1], stat[0], obj, cgi.escape(obj))
from greenlet import greenlet
objs = [obj for obj in gc.get_objects() if isinstance(obj, greenlet)]
yield " Greenlets (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj).encode("utf8")))
from Worker import Worker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Worker)]
yield " Workers (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
from Connection import Connection
objs = [obj for obj in gc.get_objects() if isinstance(obj, Connection)]
yield " Connections (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
from socket import socket
objs = [obj for obj in gc.get_objects() if isinstance(obj, socket)]
yield " Sockets (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
from msgpack import Unpacker
objs = [obj for obj in gc.get_objects() if isinstance(obj, Unpacker)]
yield " Msgpack unpacker (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
from Site import Site
objs = [obj for obj in gc.get_objects() if isinstance(obj, Site)]
yield " Sites (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
objs = [obj for obj in gc.get_objects() if isinstance(obj, self.server.log.__class__)]
yield " Loggers (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj.name)))
objs = [obj for obj in gc.get_objects() if isinstance(obj, UiRequest)]
yield " UiRequests (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
from Peer import Peer
objs = [obj for obj in gc.get_objects() if isinstance(obj, Peer)]
yield " Peers (%s): " % len(objs)
for obj in objs:
yield " - %.1fkb: %s " % (self.getObjSize(obj, hpy), cgi.escape(repr(obj)))
objs = [(key, val) for key, val in sys.modules.iteritems() if val is not None]
objs.sort()
yield " Modules (%s): " % len(objs)
for module_name, module in objs:
yield " - %.3fkb: %s %s " % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module)))
gc.collect() # Implicit grabage collection
yield "Done in %.1f" % (time.time() - s)
def actionDumpobj(self):
import gc
import sys
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
raise StopIteration
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
raise StopIteration
class_filter = self.get.get("class")
yield """
"""
objs = gc.get_objects()
for obj in objs:
obj_type = str(type(obj))
if obj_type != "" or obj.__class__.__name__ != class_filter:
continue
yield "%.1fkb %s... " % (float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)))
for attr in dir(obj):
yield "- %s: %s " % (attr, cgi.escape(str(getattr(obj, attr))))
yield " "
gc.collect() # Implicit grabage collection
def actionListobj(self):
import gc
import sys
self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
raise StopIteration
# No more if not in debug mode
if not config.debug:
yield "Not in debug mode"
raise StopIteration
type_filter = self.get.get("type")
yield """
"""
yield "Listing all %s objects in memory... " % cgi.escape(type_filter)
ref_count = {}
objs = gc.get_objects()
for obj in objs:
obj_type = str(type(obj))
if obj_type != type_filter:
continue
refs = [
ref for ref in gc.get_referrers(obj)
if hasattr(ref, "__class__") and
ref.__class__.__name__ not in ["list", "dict", "function", "type", "frame", "WeakSet", "tuple"]
]
if not refs:
continue
try:
yield "%.1fkb %s... " % (
float(sys.getsizeof(obj)) / 1024, cgi.escape(str(obj)), cgi.escape(str(obj)[0:100].ljust(100))
)
except:
continue
for ref in refs:
yield " ["
if "object at" in str(ref) or len(str(ref)) > 100:
yield str(ref.__class__.__name__)
else:
yield str(ref.__class__.__name__) + ":" + cgi.escape(str(ref))
yield "] "
ref_type = ref.__class__.__name__
if ref_type not in ref_count:
ref_count[ref_type] = [0, 0]
ref_count[ref_type][0] += 1 # Count
ref_count[ref_type][1] += float(sys.getsizeof(obj)) / 1024 # Size
yield " "
yield " Object referrer (total: %s, %.2fkb): " % (len(ref_count), sum([stat[1] for stat in ref_count.values()]))
for obj, stat in sorted(ref_count.items(), key=lambda x: x[1][0], reverse=True)[0:30]: # Sorted by count
yield " - %.1fkb = %s x %s " % (stat[1], stat[0], cgi.escape(str(obj)))
gc.collect() # Implicit grabage collection
def actionBenchmark(self):
import sys
import gc
from contextlib import contextmanager
output = self.sendHeader()
if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local:
yield "This function is disabled on this proxy"
raise StopIteration
@contextmanager
def benchmark(name, standard):
s = time.time()
output("- %s" % name)
try:
yield 1
except Exception, err:
output(" ! Error: %s " % err)
taken = time.time() - s
if taken > 0:
multipler = standard / taken
else:
multipler = 99
if multipler < 0.3:
speed = "Sloooow"
elif multipler < 0.5:
speed = "Ehh"
elif multipler < 0.8:
speed = "Goodish"
elif multipler < 1.2:
speed = "OK"
elif multipler < 1.7:
speed = "Fine"
elif multipler < 2.5:
speed = "Fast"
elif multipler < 3.5:
speed = "WOW"
else:
speed = "Insane!!"
output("%.3fs [x%.2f: %s] " % (taken, multipler, speed))
time.sleep(0.01)
yield """
"""
yield "Benchmarking ZeroNet %s (rev%s) Python %s on: %s... " % (config.version, config.rev, sys.version, sys.platform)
t = time.time()
# CryptBitcoin
yield " CryptBitcoin: "
from Crypt import CryptBitcoin
# seed = CryptBitcoin.newSeed()
# yield "- Seed: %s " % seed
seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38"
with benchmark("hdPrivatekey x 10", 0.7):
for i in range(10):
privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10)
yield "."
valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk"
assert privatekey == valid, "%s != %s" % (privatekey, valid)
data = "Hello" * 1024 # 5k
with benchmark("sign x 10", 0.35):
for i in range(10):
yield "."
sign = CryptBitcoin.sign(data, privatekey)
valid = "G1GXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOiBHB+kp4cRPZOL7l1yqK5BHa6J+W97bMjvTXtxzljp6w="
assert sign == valid, "%s != %s" % (sign, valid)
address = CryptBitcoin.privatekeyToAddress(privatekey)
if CryptBitcoin.opensslVerify: # Openssl avalible
with benchmark("openssl verify x 100", 0.37):
for i in range(100):
if i % 10 == 0:
yield "."
ok = CryptBitcoin.verify(data, address, sign)
assert ok, "does not verify from %s" % address
else:
yield " - openssl verify x 100...not avalible :( "
openssl_verify_bk = CryptBitcoin.opensslVerify # Emulate openssl not found in any way
CryptBitcoin.opensslVerify = None
with benchmark("pure-python verify x 10", 1.6):
for i in range(10):
yield "."
ok = CryptBitcoin.verify(data, address, sign)
assert ok, "does not verify from %s" % address
CryptBitcoin.opensslVerify = openssl_verify_bk
# CryptHash
yield " CryptHash: "
from Crypt import CryptHash
from cStringIO import StringIO
data = StringIO("Hello" * 1024 * 1024) # 5m
with benchmark("sha256 5M x 10", 0.6):
for i in range(10):
data.seek(0)
hash = CryptHash.sha256sum(data)
yield "."
valid = "8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa"
assert hash == valid, "%s != %s" % (hash, valid)
data = StringIO("Hello" * 1024 * 1024) # 5m
with benchmark("sha512 5M x 10", 0.6):
for i in range(10):
data.seek(0)
hash = CryptHash.sha512sum(data)
yield "."
valid = "9ca7e855d430964d5b55b114e95c6bbb114a6d478f6485df93044d87b108904d"
assert hash == valid, "%s != %s" % (hash, valid)
with benchmark("os.urandom(256) x 1000", 0.0065):
for i in range(10):
for y in range(100):
data = os.urandom(256)
yield "."
# Msgpack
import msgpack
yield " Msgpack: (version: %s) " % ".".join(map(str, msgpack.version))
binary = 'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv'
data = {"int": 1024*1024*1024, "float": 12345.67890, "text": "hello"*1024, "binary": binary}
with benchmark("pack 5K x 10 000", 0.78):
for i in range(10):
for y in range(1000):
data_packed = msgpack.packb(data)
yield "."
valid = """\x84\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa6binary\xda\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv"""
assert data_packed == valid, "%s != %s" % (repr(data_packed), repr(valid))
with benchmark("unpack 5K x 10 000", 1.2):
for i in range(10):
for y in range(1000):
data_unpacked = msgpack.unpackb(data_packed)
yield "."
assert data == data_unpacked, "%s != %s" % (data_unpacked, data)
with benchmark("streaming unpack 5K x 10 000", 1.4):
for i in range(10):
unpacker = msgpack.Unpacker()
for y in range(1000):
unpacker.feed(data_packed)
for data_unpacked in unpacker:
pass
yield "."
assert data == data_unpacked, "%s != %s" % (data_unpacked, data)
# Db
from Db import Db
import sqlite3
yield " Db: (version: %s, API: %s) " % (sqlite3.sqlite_version, sqlite3.version)
schema = {
"db_name": "TestDb",
"db_file": "%s/benchmark.db" % config.data_dir,
"maps": {
".*": {
"to_table": {
"test": "test"
}
}
},
"tables": {
"test": {
"cols": [
["test_id", "INTEGER"],
["title", "TEXT"],
["json_id", "INTEGER REFERENCES json (json_id)"]
],
"indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"],
"schema_changed": 1426195822
}
}
}
if os.path.isfile("%s/benchmark.db" % config.data_dir):
os.unlink("%s/benchmark.db" % config.data_dir)
with benchmark("Open x 10", 0.13):
for i in range(10):
db = Db(schema, "%s/benchmark.db" % config.data_dir)
db.checkTables()
db.close()
yield "."
db = Db(schema, "%s/benchmark.db" % config.data_dir)
db.checkTables()
import json
with benchmark("Insert x 10 x 1000", 1.0):
for u in range(10): # 10 user
data = {"test": []}
for i in range(1000): # 1000 line of data
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
db.updateJson("%s/test_%s.json" % (config.data_dir, u))
os.unlink("%s/test_%s.json" % (config.data_dir, u))
yield "."
with benchmark("Buffered insert x 100 x 100", 1.3):
cur = db.getCursor()
cur.execute("BEGIN")
cur.logging = False
for u in range(100, 200): # 100 user
data = {"test": []}
for i in range(100): # 1000 line of data
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
db.updateJson("%s/test_%s.json" % (config.data_dir, u), cur=cur)
os.unlink("%s/test_%s.json" % (config.data_dir, u))
if u % 10 == 0:
yield "."
cur.execute("COMMIT")
yield " - Total rows in db: %s " % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0]
with benchmark("Indexed query x 1000", 0.25):
found = 0
cur = db.getCursor()
cur.logging = False
for i in range(1000): # 1000x by test_id
res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i)
for row in res:
found += 1
if i % 100 == 0:
yield "."
assert found == 20000, "Found: %s != 20000" % found
with benchmark("Not indexed query x 100", 0.6):
found = 0
cur = db.getCursor()
cur.logging = False
for i in range(100): # 1000x by test_id
res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i)
for row in res:
found += 1
if i % 10 == 0:
yield "."
assert found == 18900, "Found: %s != 18900" % found
with benchmark("Like query x 100", 1.8):
found = 0
cur = db.getCursor()
cur.logging = False
for i in range(100): # 1000x by test_id
res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i)
for row in res:
found += 1
if i % 10 == 0:
yield "."
assert found == 38900, "Found: %s != 11000" % found
db.close()
if os.path.isfile("%s/benchmark.db" % config.data_dir):
os.unlink("%s/benchmark.db" % config.data_dir)
gc.collect() # Implicit grabage collection
# Zip
yield " Compression: "
import zipfile
test_data = "Test" * 1024
file_name = "\xc3\x81rv\xc3\xadzt\xc5\xb0r\xc5\x91t\xc3\xbck\xc3\xb6r\xc3\xb3g\xc3\xa9p\xe4\xb8\xad\xe5\x8d\x8e%s.txt"
with benchmark("Zip pack x 10", 0.12):
for i in range(10):
with zipfile.ZipFile('%s/test.zip' % config.data_dir, 'w') as archive:
for y in range(100):
zip_info = zipfile.ZipInfo(file_name % y, (1980,1,1,0,0,0))
zip_info.compress_type = zipfile.ZIP_DEFLATED
zip_info.create_system = 3
archive.writestr(zip_info, test_data)
yield "."
hash = CryptHash.sha512sum(open("%s/test.zip" % config.data_dir, "rb"))
valid = "f6ef623e6653883a1758db14aa593350e26c9dc53a8406d6e6defd6029dbd483"
assert hash == valid, "Invalid hash: %s != %s " % (hash, valid)
with benchmark("Zip unpack x 10", 0.2):
for i in range(10):
with zipfile.ZipFile('%s/test.zip' % config.data_dir) as archive:
for y in range(100):
assert archive.read(file_name % y) == test_data
yield "."
if os.path.isfile("%s/test.zip" % config.data_dir):
os.unlink("%s/test.zip" % config.data_dir)
# Tar.gz
import tarfile
import struct
# Monkey patch _init_write_gz to use fixed date in order to keep the hash independent from datetime
def nodate_write_gzip_header(self):
self.mtime = 0
original_write_gzip_header(self)
import gzip
original_write_gzip_header = gzip.GzipFile._write_gzip_header
gzip.GzipFile._write_gzip_header = nodate_write_gzip_header
test_data_io = StringIO("Test" * 1024)
with benchmark("Tar.gz pack x 10", 0.3):
for i in range(10):
with tarfile.open('%s/test.tar.gz' % config.data_dir, 'w:gz') as archive:
for y in range(100):
test_data_io.seek(0)
tar_info = tarfile.TarInfo(file_name % y)
tar_info.size = 4 * 1024
archive.addfile(tar_info, test_data_io)
yield "."
hash = CryptHash.sha512sum(open("%s/test.tar.gz" % config.data_dir, "rb"))
valid = "4704ebd8c987ed6f833059f1de9c475d443b0539b8d4c4cb8b49b26f7bbf2d19"
assert hash == valid, "Invalid hash: %s != %s " % (hash, valid)
with benchmark("Tar.gz unpack x 10", 0.2):
for i in range(10):
with tarfile.open('%s/test.tar.gz' % config.data_dir, 'r:gz') as archive:
for y in range(100):
assert archive.extractfile(file_name % y).read() == test_data
yield "."
if os.path.isfile("%s/test.tar.gz" % config.data_dir):
os.unlink("%s/test.tar.gz" % config.data_dir)
# Tar.bz2
import tarfile
test_data_io = StringIO("Test" * 1024)
with benchmark("Tar.bz2 pack x 10", 2.0):
for i in range(10):
with tarfile.open('%s/test.tar.bz2' % config.data_dir, 'w:bz2') as archive:
for y in range(100):
test_data_io.seek(0)
tar_info = tarfile.TarInfo(file_name % y)
tar_info.size = 4 * 1024
archive.addfile(tar_info, test_data_io)
yield "."
hash = CryptHash.sha512sum(open("%s/test.tar.bz2" % config.data_dir, "rb"))
valid = "90cba0b4d9abaa37b830bf37e4adba93bfd183e095b489ebee62aaa94339f3b5"
assert hash == valid, "Invalid hash: %s != %s " % (hash, valid)
with benchmark("Tar.bz2 unpack x 10", 0.5):
for i in range(10):
with tarfile.open('%s/test.tar.bz2' % config.data_dir, 'r:bz2') as archive:
for y in range(100):
assert archive.extractfile(file_name % y).read() == test_data
yield "."
if os.path.isfile("%s/test.tar.bz2" % config.data_dir):
os.unlink("%s/test.tar.bz2" % config.data_dir)
yield " Done. Total: %.2fs" % (time.time() - t)
def actionGcCollect(self):
import gc
self.sendHeader()
yield str(gc.collect())