rev101, Benchmark for crypto, sha512, sqlite, Not exits file exception fix

This commit is contained in:
HelloZeroNet 2015-04-18 03:02:08 +02:00
parent 60bfac63da
commit eea7af684d
4 changed files with 359 additions and 161 deletions

View File

@ -187,3 +187,201 @@ class UiRequestPlugin(object):
yield " - %.3fkb: %s %s<br>" % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module)))
yield "Done in %.1f" % (time.time()-s)
def actionBenchmark(self):
import sys
from contextlib import contextmanager
output = self.sendHeader()
@contextmanager
def benchmark(name, standard):
s = time.time()
output("- %s" % name)
try:
yield 1
except Exception, err:
output("<br><b>! Error: %s</b><br>" % err)
taken = time.time()-s
multipler = standard/taken
if multipler < 0.3: speed = "Sloooow"
elif multipler < 0.5: speed = "Ehh"
elif multipler < 0.8: speed = "Goodish"
elif multipler < 1.2: speed = "OK"
elif multipler < 1.7: speed = "Fine"
elif multipler < 2.5: speed = "Fast"
elif multipler < 3.5: speed = "WOW"
else: speed = "Insane!!"
output("%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed))
time.sleep(0.01)
yield """
<style>
* { font-family: monospace }
table * { text-align: right; padding: 0px 10px }
</style>
"""
yield "Benchmarking ZeroNet %s (rev%s) Python %s, platform: %s...<br>" % (config.version, config.rev, sys.version, sys.platform)
t = time.time()
yield "<br>CryptBitcoin:<br>"
from Crypt import CryptBitcoin
# seed = CryptBitcoin.newSeed()
# yield "- Seed: %s<br>" % seed
seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38"
with benchmark("hdPrivatekey x 10", 0.7):
for i in range(10):
privatekey = CryptBitcoin.hdPrivatekey(seed, i*10)
yield "."
valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk"
assert privatekey == valid, "%s != %s" % (privatekey, valid)
data = "Hello"*1024 #5k
with benchmark("sign x 10", 0.35):
for i in range(10):
yield "."
sign = CryptBitcoin.sign(data, privatekey)
valid = "HFGXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOi+4+BbWHjuwmx0EaKNV1G+kP0tQDxWu0YApxwxZbSmZU="
assert sign == valid, "%s != %s" % (sign, valid)
address = CryptBitcoin.privatekeyToAddress(privatekey)
with benchmark("verify x 10", 1.6):
for i in range(10):
yield "."
ok = CryptBitcoin.verify(data, address, sign)
assert ok, "does not verify from %s" % address
yield "<br>CryptHash:<br>"
from Crypt import CryptHash
from cStringIO import StringIO
data = StringIO("Hello"*1024*1024) #5m
with benchmark("sha512 x 10 000", 1):
for i in range(10):
for y in range(10000):
hash = CryptHash.sha512sum(data)
yield "."
valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce"
assert hash == valid, "%s != %s" % (hash, valid)
yield "<br>Db:<br>"
from Db import Db
schema = {
"db_name": "TestDb",
"db_file": "data/benchmark.db",
"maps": {
".*": {
"to_table": {
"test": "test"
}
}
},
"tables": {
"test": {
"cols": [
["test_id", "INTEGER"],
["title", "TEXT"],
["json_id", "INTEGER REFERENCES json (json_id)"]
],
"indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"],
"schema_changed": 1426195822
}
}
}
if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db")
with benchmark("Open x 10", 0.13):
for i in range(10):
db = Db(schema, "data/benchmark.db")
db.checkTables()
db.close()
yield "."
db = Db(schema, "data/benchmark.db")
db.checkTables()
import json
with benchmark("Insert x 10 x 1000", 1.0):
for u in range(10): # 10 user
data = {"test": []}
for i in range(1000): # 1000 line of data
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
json.dump(data, open("data/test_%s.json" % u, "w"))
db.loadJson("data/test_%s.json" % u)
os.unlink("data/test_%s.json" % u)
yield "."
with benchmark("Buffered insert x 100 x 100", 1.3):
cur = db.getCursor()
cur.execute("BEGIN")
cur.logging = False
for u in range(100, 200): # 100 user
data = {"test": []}
for i in range(100): # 1000 line of data
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
json.dump(data, open("data/test_%s.json" % u, "w"))
db.loadJson("data/test_%s.json" % u, cur=cur)
os.unlink("data/test_%s.json" % u)
if u%10 == 0: yield "."
cur.execute("COMMIT")
yield " - Total rows in db: %s<br>" % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0]
with benchmark("Indexed query x 1000", 0.25):
found = 0
cur = db.getCursor()
cur.logging = False
for i in range(1000): # 1000x by test_id
res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i)
for row in res:
found += 1
if i%100 == 0: yield "."
assert found == 20000, "Found: %s != 20000" % found
with benchmark("Not indexed query x 100", 0.6):
found = 0
cur = db.getCursor()
cur.logging = False
for i in range(100): # 1000x by test_id
res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i)
for row in res:
found += 1
if i%10 == 0: yield "."
assert found == 18900, "Found: %s != 18900" % found
with benchmark("Like query x 100", 1.8):
found = 0
cur = db.getCursor()
cur.logging = False
for i in range(100): # 1000x by test_id
res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i)
for row in res:
found += 1
if i%10 == 0: yield "."
assert found == 38900, "Found: %s != 11000" % found
db.close()
if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db")
yield "<br>Done. Total: %.2fs" % (time.time()-t)

View File

@ -1,158 +1,158 @@
import argparse, sys, os, time
import ConfigParser
class Config(object):
def __init__(self):
self.version = "0.2.9"
self.rev = 100
self.parser = self.createArguments()
argv = sys.argv[:] # Copy command line arguments
argv = self.parseConfig(argv) # Add arguments from config file
self.parseCommandline(argv) # Parse argv
self.setAttributes()
def __str__(self):
return str(self.arguments).replace("Namespace", "Config") # Using argparse str output
# Create command line arguments
def createArguments(self):
# Platform specific
if sys.platform.startswith("win"):
coffeescript = "type %s | tools\\coffee\\coffee.cmd"
else:
coffeescript = None
# Create parser
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
subparsers = parser.add_subparsers(title="Action to perform", dest="action")
# Main
action = subparsers.add_parser("main", help='Start UiServer and FileServer (default)')
# SiteCreate
action = subparsers.add_parser("siteCreate", help='Create a new site')
# SiteSign
action = subparsers.add_parser("siteSign", help='Update and sign content.json: address [privatekey]')
action.add_argument('address', help='Site to sign')
action.add_argument('privatekey', help='Private key (default: ask on execute)', nargs='?')
action.add_argument('--inner_path', help='File you want to sign (default: content.json)', default="content.json", metavar="inner_path")
# SitePublish
action = subparsers.add_parser("sitePublish", help='Publish site to other peers: address')
action.add_argument('address', help='Site to publish')
action.add_argument('peer_ip', help='Peer ip to publish (default: random peers ip from tracker)', default=None, nargs='?')
action.add_argument('peer_port', help='Peer port to publish (default: random peer port from tracker)', default=15441, nargs='?')
action.add_argument('--inner_path', help='Content.json you want to publish (default: content.json)', default="content.json", metavar="inner_path")
# SiteVerify
action = subparsers.add_parser("siteVerify", help='Verify site files using sha512: address')
action.add_argument('address', help='Site to verify')
#dbRebuild
action = subparsers.add_parser("dbRebuild", help='Rebuild site database cache')
action.add_argument('address', help='Site to rebuild')
#dbQuery
action = subparsers.add_parser("dbQuery", help='Query site sql cache')
action.add_argument('address', help='Site to query')
action.add_argument('query', help='Sql query')
# PeerPing
action = subparsers.add_parser("peerPing", help='Send Ping command to peer')
action.add_argument('peer_ip', help='Peer ip')
action.add_argument('peer_port', help='Peer port')
# PeerGetFile
action = subparsers.add_parser("peerGetFile", help='Request and print a file content from peer')
action.add_argument('peer_ip', help='Peer ip')
action.add_argument('peer_port', help='Peer port')
action.add_argument('site', help='Site address')
action.add_argument('filename', help='File name to request')
# Config parameters
parser.add_argument('--debug', help='Debug mode', action='store_true')
parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true')
parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip')
parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port')
parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip')
parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name')
parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit')
parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip')
parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port')
parser.add_argument('--disable_zeromq', help='Disable compatibility with old clients', action='store_true')
parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip')
parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, metavar='executable_path')
parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev))
return parser
# Find arguments specificed for current action
def getActionArguments(self):
back = {}
arguments = self.parser._subparsers._group_actions[0].choices[self.action]._actions[1:] # First is --version
for argument in arguments:
back[argument.dest] = getattr(self, argument.dest)
return back
# Try to find action from sys.argv
def getAction(self, argv):
actions = [action.choices.keys() for action in self.parser._actions if action.dest == "action"][0] # Valid actions
found_action = False
for action in actions: # See if any in sys.argv
if action in argv:
found_action = action
break
return found_action
# Parse command line arguments
def parseCommandline(self, argv):
# Find out if action is specificed on start
action = self.getAction(argv)
if len(argv) == 1 or not action: # If no action specificed set the main action
argv.append("main")
if "zeronet.py" in argv[0]:
self.arguments = self.parser.parse_args(argv[1:])
else: # Silent errors if not started with zeronet.py
self.arguments = self.parser.parse_args(argv[1:])
# Parse config file
def parseConfig(self, argv):
if os.path.isfile("zeronet.conf"):
config = ConfigParser.ConfigParser(allow_no_value=True)
config.read('zeronet.conf')
for section in config.sections():
for key, val in config.items(section):
if section != "global": # If not global prefix key with section
key = section+"_"+key
if val: argv.insert(1, val)
argv.insert(1, "--%s" % key)
return argv
# Expose arguments as class attributes
def setAttributes(self):
# Set attributes from arguments
args = vars(self.arguments)
for key, val in args.items():
setattr(self, key, val)
config = Config()
import argparse, sys, os, time
import ConfigParser
class Config(object):
def __init__(self):
self.version = "0.2.9"
self.rev = 101
self.parser = self.createArguments()
argv = sys.argv[:] # Copy command line arguments
argv = self.parseConfig(argv) # Add arguments from config file
self.parseCommandline(argv) # Parse argv
self.setAttributes()
def __str__(self):
return str(self.arguments).replace("Namespace", "Config") # Using argparse str output
# Create command line arguments
def createArguments(self):
# Platform specific
if sys.platform.startswith("win"):
coffeescript = "type %s | tools\\coffee\\coffee.cmd"
else:
coffeescript = None
# Create parser
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
subparsers = parser.add_subparsers(title="Action to perform", dest="action")
# Main
action = subparsers.add_parser("main", help='Start UiServer and FileServer (default)')
# SiteCreate
action = subparsers.add_parser("siteCreate", help='Create a new site')
# SiteSign
action = subparsers.add_parser("siteSign", help='Update and sign content.json: address [privatekey]')
action.add_argument('address', help='Site to sign')
action.add_argument('privatekey', help='Private key (default: ask on execute)', nargs='?')
action.add_argument('--inner_path', help='File you want to sign (default: content.json)', default="content.json", metavar="inner_path")
# SitePublish
action = subparsers.add_parser("sitePublish", help='Publish site to other peers: address')
action.add_argument('address', help='Site to publish')
action.add_argument('peer_ip', help='Peer ip to publish (default: random peers ip from tracker)', default=None, nargs='?')
action.add_argument('peer_port', help='Peer port to publish (default: random peer port from tracker)', default=15441, nargs='?')
action.add_argument('--inner_path', help='Content.json you want to publish (default: content.json)', default="content.json", metavar="inner_path")
# SiteVerify
action = subparsers.add_parser("siteVerify", help='Verify site files using sha512: address')
action.add_argument('address', help='Site to verify')
#dbRebuild
action = subparsers.add_parser("dbRebuild", help='Rebuild site database cache')
action.add_argument('address', help='Site to rebuild')
#dbQuery
action = subparsers.add_parser("dbQuery", help='Query site sql cache')
action.add_argument('address', help='Site to query')
action.add_argument('query', help='Sql query')
# PeerPing
action = subparsers.add_parser("peerPing", help='Send Ping command to peer')
action.add_argument('peer_ip', help='Peer ip')
action.add_argument('peer_port', help='Peer port')
# PeerGetFile
action = subparsers.add_parser("peerGetFile", help='Request and print a file content from peer')
action.add_argument('peer_ip', help='Peer ip')
action.add_argument('peer_port', help='Peer port')
action.add_argument('site', help='Site address')
action.add_argument('filename', help='File name to request')
# Config parameters
parser.add_argument('--debug', help='Debug mode', action='store_true')
parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true')
parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip')
parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port')
parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip')
parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name')
parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit')
parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip')
parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port')
parser.add_argument('--disable_zeromq', help='Disable compatibility with old clients', action='store_true')
parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip')
parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, metavar='executable_path')
parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev))
return parser
# Find arguments specificed for current action
def getActionArguments(self):
back = {}
arguments = self.parser._subparsers._group_actions[0].choices[self.action]._actions[1:] # First is --version
for argument in arguments:
back[argument.dest] = getattr(self, argument.dest)
return back
# Try to find action from sys.argv
def getAction(self, argv):
actions = [action.choices.keys() for action in self.parser._actions if action.dest == "action"][0] # Valid actions
found_action = False
for action in actions: # See if any in sys.argv
if action in argv:
found_action = action
break
return found_action
# Parse command line arguments
def parseCommandline(self, argv):
# Find out if action is specificed on start
action = self.getAction(argv)
if len(argv) == 1 or not action: # If no action specificed set the main action
argv.append("main")
if "zeronet.py" in argv[0]:
self.arguments = self.parser.parse_args(argv[1:])
else: # Silent errors if not started with zeronet.py
self.arguments = self.parser.parse_args(argv[1:])
# Parse config file
def parseConfig(self, argv):
if os.path.isfile("zeronet.conf"):
config = ConfigParser.ConfigParser(allow_no_value=True)
config.read('zeronet.conf')
for section in config.sections():
for key, val in config.items(section):
if section != "global": # If not global prefix key with section
key = section+"_"+key
if val: argv.insert(1, val)
argv.insert(1, "--%s" % key)
return argv
# Expose arguments as class attributes
def setAttributes(self):
# Set attributes from arguments
args = vars(self.arguments)
for key, val in args.items():
setattr(self, key, val)
config = Config()

View File

@ -33,7 +33,7 @@ class Db:
# Execute query using dbcursor
def execute(self, query, params):
def execute(self, query, params = None):
if not self.conn: self.connect()
return self.cur.execute(query, params)

View File

@ -102,7 +102,7 @@ class UiRequest(object):
headers.append(("Content-Type", content_type))
for extra_header in extra_headers:
headers.append(extra_header)
self.start_response(status_texts[status], headers)
return self.start_response(status_texts[status], headers)
# Renders a template
@ -224,7 +224,7 @@ class UiRequest(object):
return self.actionFile(file_path)
else:
self.log.debug("File not found: %s" % match.group("inner_path"))
self.error404(match.group("inner_path"))
return self.error404(match.group("inner_path"))
#self.sendHeader(404)
#return "Not found"