From eea7af684dd712009b049c2f92e28680d7714124 Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Sat, 18 Apr 2015 03:02:08 +0200 Subject: [PATCH] rev101, Benchmark for crypto, sha512, sqlite, Not exits file exception fix --- plugins/Stats/StatsPlugin.py | 198 ++++++++++++++++++++++ src/Config.py | 316 +++++++++++++++++------------------ src/Db/Db.py | 2 +- src/Ui/UiRequest.py | 4 +- 4 files changed, 359 insertions(+), 161 deletions(-) diff --git a/plugins/Stats/StatsPlugin.py b/plugins/Stats/StatsPlugin.py index 3f25c05e..a3df14e6 100644 --- a/plugins/Stats/StatsPlugin.py +++ b/plugins/Stats/StatsPlugin.py @@ -187,3 +187,201 @@ class UiRequestPlugin(object): yield " - %.3fkb: %s %s
" % (self.getObjSize(module, hpy), module_name, cgi.escape(repr(module))) yield "Done in %.1f" % (time.time()-s) + + + def actionBenchmark(self): + import sys + from contextlib import contextmanager + + output = self.sendHeader() + + @contextmanager + def benchmark(name, standard): + s = time.time() + output("- %s" % name) + try: + yield 1 + except Exception, err: + output("
! Error: %s
" % err) + taken = time.time()-s + multipler = standard/taken + if multipler < 0.3: speed = "Sloooow" + elif multipler < 0.5: speed = "Ehh" + elif multipler < 0.8: speed = "Goodish" + elif multipler < 1.2: speed = "OK" + elif multipler < 1.7: speed = "Fine" + elif multipler < 2.5: speed = "Fast" + elif multipler < 3.5: speed = "WOW" + else: speed = "Insane!!" + output("%.3fs [x%.2f: %s]
" % (taken, multipler, speed)) + time.sleep(0.01) + + + yield """ + + """ + + yield "Benchmarking ZeroNet %s (rev%s) Python %s, platform: %s...
" % (config.version, config.rev, sys.version, sys.platform) + + t = time.time() + + yield "
CryptBitcoin:
" + from Crypt import CryptBitcoin + + # seed = CryptBitcoin.newSeed() + # yield "- Seed: %s
" % seed + seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38" + + with benchmark("hdPrivatekey x 10", 0.7): + for i in range(10): + privatekey = CryptBitcoin.hdPrivatekey(seed, i*10) + yield "." + valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk" + assert privatekey == valid, "%s != %s" % (privatekey, valid) + + + data = "Hello"*1024 #5k + with benchmark("sign x 10", 0.35): + for i in range(10): + yield "." + sign = CryptBitcoin.sign(data, privatekey) + valid = "HFGXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOi+4+BbWHjuwmx0EaKNV1G+kP0tQDxWu0YApxwxZbSmZU=" + assert sign == valid, "%s != %s" % (sign, valid) + + + address = CryptBitcoin.privatekeyToAddress(privatekey) + with benchmark("verify x 10", 1.6): + for i in range(10): + yield "." + ok = CryptBitcoin.verify(data, address, sign) + assert ok, "does not verify from %s" % address + + + yield "
CryptHash:
" + from Crypt import CryptHash + from cStringIO import StringIO + + data = StringIO("Hello"*1024*1024) #5m + with benchmark("sha512 x 10 000", 1): + for i in range(10): + for y in range(10000): + hash = CryptHash.sha512sum(data) + yield "." + valid = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce" + assert hash == valid, "%s != %s" % (hash, valid) + + + yield "
Db:
" + from Db import Db + + schema = { + "db_name": "TestDb", + "db_file": "data/benchmark.db", + "maps": { + ".*": { + "to_table": { + "test": "test" + } + } + }, + "tables": { + "test": { + "cols": [ + ["test_id", "INTEGER"], + ["title", "TEXT"], + ["json_id", "INTEGER REFERENCES json (json_id)"] + ], + "indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"], + "schema_changed": 1426195822 + } + } + } + + if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") + + with benchmark("Open x 10", 0.13): + for i in range(10): + db = Db(schema, "data/benchmark.db") + db.checkTables() + db.close() + yield "." + + + db = Db(schema, "data/benchmark.db") + db.checkTables() + import json + + with benchmark("Insert x 10 x 1000", 1.0): + for u in range(10): # 10 user + data = {"test": []} + for i in range(1000): # 1000 line of data + data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) + json.dump(data, open("data/test_%s.json" % u, "w")) + db.loadJson("data/test_%s.json" % u) + os.unlink("data/test_%s.json" % u) + yield "." + + + with benchmark("Buffered insert x 100 x 100", 1.3): + cur = db.getCursor() + cur.execute("BEGIN") + cur.logging = False + for u in range(100, 200): # 100 user + data = {"test": []} + for i in range(100): # 1000 line of data + data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) + json.dump(data, open("data/test_%s.json" % u, "w")) + db.loadJson("data/test_%s.json" % u, cur=cur) + os.unlink("data/test_%s.json" % u) + if u%10 == 0: yield "." + cur.execute("COMMIT") + + yield " - Total rows in db: %s
" % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0] + + with benchmark("Indexed query x 1000", 0.25): + found = 0 + cur = db.getCursor() + cur.logging = False + for i in range(1000): # 1000x by test_id + res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i) + for row in res: + found += 1 + if i%100 == 0: yield "." + + assert found == 20000, "Found: %s != 20000" % found + + + with benchmark("Not indexed query x 100", 0.6): + found = 0 + cur = db.getCursor() + cur.logging = False + for i in range(100): # 1000x by test_id + res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i) + for row in res: + found += 1 + if i%10 == 0: yield "." + + assert found == 18900, "Found: %s != 18900" % found + + + with benchmark("Like query x 100", 1.8): + found = 0 + cur = db.getCursor() + cur.logging = False + for i in range(100): # 1000x by test_id + res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i) + for row in res: + found += 1 + if i%10 == 0: yield "." + + assert found == 38900, "Found: %s != 11000" % found + + + db.close() + if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") + + + yield "
Done. Total: %.2fs" % (time.time()-t) \ No newline at end of file diff --git a/src/Config.py b/src/Config.py index 126c1550..9d4d05cf 100644 --- a/src/Config.py +++ b/src/Config.py @@ -1,158 +1,158 @@ -import argparse, sys, os, time -import ConfigParser - -class Config(object): - def __init__(self): - self.version = "0.2.9" - self.rev = 100 - self.parser = self.createArguments() - argv = sys.argv[:] # Copy command line arguments - argv = self.parseConfig(argv) # Add arguments from config file - self.parseCommandline(argv) # Parse argv - self.setAttributes() - - - def __str__(self): - return str(self.arguments).replace("Namespace", "Config") # Using argparse str output - - - # Create command line arguments - def createArguments(self): - # Platform specific - if sys.platform.startswith("win"): - coffeescript = "type %s | tools\\coffee\\coffee.cmd" - else: - coffeescript = None - - # Create parser - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - subparsers = parser.add_subparsers(title="Action to perform", dest="action") - - # Main - action = subparsers.add_parser("main", help='Start UiServer and FileServer (default)') - - # SiteCreate - action = subparsers.add_parser("siteCreate", help='Create a new site') - - # SiteSign - action = subparsers.add_parser("siteSign", help='Update and sign content.json: address [privatekey]') - action.add_argument('address', help='Site to sign') - action.add_argument('privatekey', help='Private key (default: ask on execute)', nargs='?') - action.add_argument('--inner_path', help='File you want to sign (default: content.json)', default="content.json", metavar="inner_path") - - # SitePublish - action = subparsers.add_parser("sitePublish", help='Publish site to other peers: address') - action.add_argument('address', help='Site to publish') - action.add_argument('peer_ip', help='Peer ip to publish (default: random peers ip from tracker)', default=None, nargs='?') - action.add_argument('peer_port', help='Peer port to publish (default: random peer port from tracker)', default=15441, nargs='?') - action.add_argument('--inner_path', help='Content.json you want to publish (default: content.json)', default="content.json", metavar="inner_path") - - # SiteVerify - action = subparsers.add_parser("siteVerify", help='Verify site files using sha512: address') - action.add_argument('address', help='Site to verify') - - #dbRebuild - action = subparsers.add_parser("dbRebuild", help='Rebuild site database cache') - action.add_argument('address', help='Site to rebuild') - - #dbQuery - action = subparsers.add_parser("dbQuery", help='Query site sql cache') - action.add_argument('address', help='Site to query') - action.add_argument('query', help='Sql query') - - # PeerPing - action = subparsers.add_parser("peerPing", help='Send Ping command to peer') - action.add_argument('peer_ip', help='Peer ip') - action.add_argument('peer_port', help='Peer port') - - # PeerGetFile - action = subparsers.add_parser("peerGetFile", help='Request and print a file content from peer') - action.add_argument('peer_ip', help='Peer ip') - action.add_argument('peer_port', help='Peer port') - action.add_argument('site', help='Site address') - action.add_argument('filename', help='File name to request') - - - - # Config parameters - parser.add_argument('--debug', help='Debug mode', action='store_true') - parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true') - - parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') - parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port') - parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip') - parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name') - parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address') - parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit') - - parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') - parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port') - parser.add_argument('--disable_zeromq', help='Disable compatibility with old clients', action='store_true') - parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') - parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') - parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip') - - parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, metavar='executable_path') - - parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev)) - - return parser - - - # Find arguments specificed for current action - def getActionArguments(self): - back = {} - arguments = self.parser._subparsers._group_actions[0].choices[self.action]._actions[1:] # First is --version - for argument in arguments: - back[argument.dest] = getattr(self, argument.dest) - return back - - - - # Try to find action from sys.argv - def getAction(self, argv): - actions = [action.choices.keys() for action in self.parser._actions if action.dest == "action"][0] # Valid actions - found_action = False - for action in actions: # See if any in sys.argv - if action in argv: - found_action = action - break - return found_action - - - # Parse command line arguments - def parseCommandline(self, argv): - # Find out if action is specificed on start - action = self.getAction(argv) - if len(argv) == 1 or not action: # If no action specificed set the main action - argv.append("main") - if "zeronet.py" in argv[0]: - self.arguments = self.parser.parse_args(argv[1:]) - else: # Silent errors if not started with zeronet.py - self.arguments = self.parser.parse_args(argv[1:]) - - - # Parse config file - def parseConfig(self, argv): - if os.path.isfile("zeronet.conf"): - config = ConfigParser.ConfigParser(allow_no_value=True) - config.read('zeronet.conf') - for section in config.sections(): - for key, val in config.items(section): - if section != "global": # If not global prefix key with section - key = section+"_"+key - if val: argv.insert(1, val) - argv.insert(1, "--%s" % key) - return argv - - - - # Expose arguments as class attributes - def setAttributes(self): - # Set attributes from arguments - args = vars(self.arguments) - for key, val in args.items(): - setattr(self, key, val) - - -config = Config() +import argparse, sys, os, time +import ConfigParser + +class Config(object): + def __init__(self): + self.version = "0.2.9" + self.rev = 101 + self.parser = self.createArguments() + argv = sys.argv[:] # Copy command line arguments + argv = self.parseConfig(argv) # Add arguments from config file + self.parseCommandline(argv) # Parse argv + self.setAttributes() + + + def __str__(self): + return str(self.arguments).replace("Namespace", "Config") # Using argparse str output + + + # Create command line arguments + def createArguments(self): + # Platform specific + if sys.platform.startswith("win"): + coffeescript = "type %s | tools\\coffee\\coffee.cmd" + else: + coffeescript = None + + # Create parser + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + subparsers = parser.add_subparsers(title="Action to perform", dest="action") + + # Main + action = subparsers.add_parser("main", help='Start UiServer and FileServer (default)') + + # SiteCreate + action = subparsers.add_parser("siteCreate", help='Create a new site') + + # SiteSign + action = subparsers.add_parser("siteSign", help='Update and sign content.json: address [privatekey]') + action.add_argument('address', help='Site to sign') + action.add_argument('privatekey', help='Private key (default: ask on execute)', nargs='?') + action.add_argument('--inner_path', help='File you want to sign (default: content.json)', default="content.json", metavar="inner_path") + + # SitePublish + action = subparsers.add_parser("sitePublish", help='Publish site to other peers: address') + action.add_argument('address', help='Site to publish') + action.add_argument('peer_ip', help='Peer ip to publish (default: random peers ip from tracker)', default=None, nargs='?') + action.add_argument('peer_port', help='Peer port to publish (default: random peer port from tracker)', default=15441, nargs='?') + action.add_argument('--inner_path', help='Content.json you want to publish (default: content.json)', default="content.json", metavar="inner_path") + + # SiteVerify + action = subparsers.add_parser("siteVerify", help='Verify site files using sha512: address') + action.add_argument('address', help='Site to verify') + + #dbRebuild + action = subparsers.add_parser("dbRebuild", help='Rebuild site database cache') + action.add_argument('address', help='Site to rebuild') + + #dbQuery + action = subparsers.add_parser("dbQuery", help='Query site sql cache') + action.add_argument('address', help='Site to query') + action.add_argument('query', help='Sql query') + + # PeerPing + action = subparsers.add_parser("peerPing", help='Send Ping command to peer') + action.add_argument('peer_ip', help='Peer ip') + action.add_argument('peer_port', help='Peer port') + + # PeerGetFile + action = subparsers.add_parser("peerGetFile", help='Request and print a file content from peer') + action.add_argument('peer_ip', help='Peer ip') + action.add_argument('peer_port', help='Peer port') + action.add_argument('site', help='Site address') + action.add_argument('filename', help='File name to request') + + + + # Config parameters + parser.add_argument('--debug', help='Debug mode', action='store_true') + parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true') + + parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') + parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port') + parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip') + parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name') + parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address') + parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit') + + parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') + parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port') + parser.add_argument('--disable_zeromq', help='Disable compatibility with old clients', action='store_true') + parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port') + parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true') + parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip') + + parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript, metavar='executable_path') + + parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev)) + + return parser + + + # Find arguments specificed for current action + def getActionArguments(self): + back = {} + arguments = self.parser._subparsers._group_actions[0].choices[self.action]._actions[1:] # First is --version + for argument in arguments: + back[argument.dest] = getattr(self, argument.dest) + return back + + + + # Try to find action from sys.argv + def getAction(self, argv): + actions = [action.choices.keys() for action in self.parser._actions if action.dest == "action"][0] # Valid actions + found_action = False + for action in actions: # See if any in sys.argv + if action in argv: + found_action = action + break + return found_action + + + # Parse command line arguments + def parseCommandline(self, argv): + # Find out if action is specificed on start + action = self.getAction(argv) + if len(argv) == 1 or not action: # If no action specificed set the main action + argv.append("main") + if "zeronet.py" in argv[0]: + self.arguments = self.parser.parse_args(argv[1:]) + else: # Silent errors if not started with zeronet.py + self.arguments = self.parser.parse_args(argv[1:]) + + + # Parse config file + def parseConfig(self, argv): + if os.path.isfile("zeronet.conf"): + config = ConfigParser.ConfigParser(allow_no_value=True) + config.read('zeronet.conf') + for section in config.sections(): + for key, val in config.items(section): + if section != "global": # If not global prefix key with section + key = section+"_"+key + if val: argv.insert(1, val) + argv.insert(1, "--%s" % key) + return argv + + + + # Expose arguments as class attributes + def setAttributes(self): + # Set attributes from arguments + args = vars(self.arguments) + for key, val in args.items(): + setattr(self, key, val) + + +config = Config() diff --git a/src/Db/Db.py b/src/Db/Db.py index 34dee0ba..9a357a1e 100644 --- a/src/Db/Db.py +++ b/src/Db/Db.py @@ -33,7 +33,7 @@ class Db: # Execute query using dbcursor - def execute(self, query, params): + def execute(self, query, params = None): if not self.conn: self.connect() return self.cur.execute(query, params) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 59946761..fb080b79 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -102,7 +102,7 @@ class UiRequest(object): headers.append(("Content-Type", content_type)) for extra_header in extra_headers: headers.append(extra_header) - self.start_response(status_texts[status], headers) + return self.start_response(status_texts[status], headers) # Renders a template @@ -224,7 +224,7 @@ class UiRequest(object): return self.actionFile(file_path) else: self.log.debug("File not found: %s" % match.group("inner_path")) - self.error404(match.group("inner_path")) + return self.error404(match.group("inner_path")) #self.sendHeader(404) #return "Not found"