From 94165176d31474663043c1237cad68f6421cb622 Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Wed, 27 May 2015 16:58:36 +0200 Subject: [PATCH 1/2] rev194, Fix for ugly openSSL layer memory leak, memory usage test to openssl stability test --- src/Config.py | 2 +- src/lib/opensslVerify/opensslVerify-alter2.py | 19 ++- src/lib/opensslVerify/opensslVerify.py | 114 +++++++++++------- src/lib/opensslVerify/stablityTest.py | 27 ++++- zeronet.py | 2 +- 5 files changed, 111 insertions(+), 53 deletions(-) diff --git a/src/Config.py b/src/Config.py index 1acd6d13..834e1630 100644 --- a/src/Config.py +++ b/src/Config.py @@ -4,7 +4,7 @@ import ConfigParser class Config(object): def __init__(self): self.version = "0.3.0" - self.rev = 193 + self.rev = 194 self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file diff --git a/src/lib/opensslVerify/opensslVerify-alter2.py b/src/lib/opensslVerify/opensslVerify-alter2.py index 69b607e2..e0406d75 100644 --- a/src/lib/opensslVerify/opensslVerify-alter2.py +++ b/src/lib/opensslVerify/opensslVerify-alter2.py @@ -74,9 +74,24 @@ def public_key_to_bc_address(public_key): h160 = hash_160(public_key) return hash_160_to_bc_address(h160) +def encode(val, base, minlen=0): + base, minlen = int(base), int(minlen) + code_string = ''.join([chr(x) for x in range(256)]) + result = "" + while val > 0: + result = code_string[val % base] + result + val //= base + return code_string[0] * max(minlen - len(result), 0) + result + +def num_to_var_int(x): + x = int(x) + if x < 253: return chr(x) + elif x < 65536: return chr(253)+encode(x, 256, 2)[::-1] + elif x < 4294967296: return chr(254) + encode(x, 256, 4)[::-1] + else: return chr(255) + encode(x, 256, 8)[::-1] + def msg_magic(message): - #return "\x18Bitcoin Signed Message:\n" + chr( len(message) ) + message - return "\x18Bitcoin Signed Message:\n" + chr( len(message) ) + message + return "\x18Bitcoin Signed Message:\n" + num_to_var_int( len(message) ) + message def get_address(eckey): size = ssl.i2o_ECPublicKey (eckey, 0) diff --git a/src/lib/opensslVerify/opensslVerify.py b/src/lib/opensslVerify/opensslVerify.py index c7736a2e..3ace4db0 100644 --- a/src/lib/opensslVerify/opensslVerify.py +++ b/src/lib/opensslVerify/opensslVerify.py @@ -10,6 +10,8 @@ import ctypes import ctypes.util import hashlib import base64 +import time +import logging addrtype = 0 class _OpenSSL: @@ -17,6 +19,7 @@ class _OpenSSL: Wrapper for OpenSSL using ctypes """ def __init__(self, library): + self.time_opened = time.time() """ Build the wrapper """ @@ -172,14 +175,23 @@ class _OpenSSL: self.i2o_ECPublicKey.restype = ctypes.c_void_p self.i2o_ECPublicKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p] + self.BN_CTX_free = self._lib.BN_CTX_free + self.BN_CTX_free.restype = None + self.BN_CTX_free.argtypes = [ctypes.c_void_p] + + self.EC_POINT_free = self._lib.EC_POINT_free + self.EC_POINT_free.restype = None + self.EC_POINT_free.argtypes = [ctypes.c_void_p] +def openLibrary(): + global ssl + try: + ssl = _OpenSSL("src/lib/opensslVerify/libeay32.dll") + except: + ssl = _OpenSSL(ctypes.util.find_library('ssl') or ctypes.util.find_library('crypto') or 'libeay32') -try: - ssl = _OpenSSL("src/lib/opensslVerify/libeay32.dll") -except: - ssl = _OpenSSL(ctypes.util.find_library('ssl') or ctypes.util.find_library('crypto') or 'libeay32') - +openLibrary() openssl_version = "%.9X" % ssl._lib.SSLeay() NID_secp256k1 = 714 @@ -296,51 +308,58 @@ def SetCompactSignature(pkey, hash, signature): def ECDSA_SIG_recover_key_GFp(eckey, r, s, msg, msglen, recid, check): n = 0 i = recid / 2 + ctx = R = O = Q = None - group = ssl.EC_KEY_get0_group(eckey) - ctx = ssl.BN_CTX_new() - ssl.BN_CTX_start(ctx) - order = ssl.BN_CTX_get(ctx) - ssl.EC_GROUP_get_order(group, order, ctx) - x = ssl.BN_CTX_get(ctx) - ssl.BN_copy(x, order); - ssl.BN_mul_word(x, i); - ssl.BN_add(x, x, r) - field = ssl.BN_CTX_get(ctx) - ssl.EC_GROUP_get_curve_GFp(group, field, None, None, ctx) + try: + group = ssl.EC_KEY_get0_group(eckey) + ctx = ssl.BN_CTX_new() + ssl.BN_CTX_start(ctx) + order = ssl.BN_CTX_get(ctx) + ssl.EC_GROUP_get_order(group, order, ctx) + x = ssl.BN_CTX_get(ctx) + ssl.BN_copy(x, order); + ssl.BN_mul_word(x, i); + ssl.BN_add(x, x, r) + field = ssl.BN_CTX_get(ctx) + ssl.EC_GROUP_get_curve_GFp(group, field, None, None, ctx) - if (ssl.BN_cmp(x, field) >= 0): - return False - - R = ssl.EC_POINT_new(group) - ssl.EC_POINT_set_compressed_coordinates_GFp(group, R, x, recid % 2, ctx) - - if check: - O = ssl.EC_POINT_new(group) - ssl.EC_POINT_mul(group, O, None, R, order, ctx) - if ssl.EC_POINT_is_at_infinity(group, O): + if (ssl.BN_cmp(x, field) >= 0): return False - Q = ssl.EC_POINT_new(group) - n = ssl.EC_GROUP_get_degree(group) - e = ssl.BN_CTX_get(ctx) - ssl.BN_bin2bn(msg, msglen, e) - if 8 * msglen > n: ssl.BN_rshift(e, e, 8 - (n & 7)) + R = ssl.EC_POINT_new(group) + ssl.EC_POINT_set_compressed_coordinates_GFp(group, R, x, recid % 2, ctx) - zero = ssl.BN_CTX_get(ctx) - ssl.BN_set_word(zero, 0) - ssl.BN_mod_sub(e, zero, e, order, ctx) - rr = ssl.BN_CTX_get(ctx); - ssl.BN_mod_inverse(rr, r, order, ctx) - sor = ssl.BN_CTX_get(ctx) - ssl.BN_mod_mul(sor, s, rr, order, ctx) - eor = ssl.BN_CTX_get(ctx) - ssl.BN_mod_mul(eor, e, rr, order, ctx) - ssl.EC_POINT_mul(group, Q, eor, R, sor, ctx) - ssl.EC_KEY_set_public_key(eckey, Q) - return eckey + if check: + O = ssl.EC_POINT_new(group) + ssl.EC_POINT_mul(group, O, None, R, order, ctx) + if ssl.EC_POINT_is_at_infinity(group, O): + return False -def close(): + Q = ssl.EC_POINT_new(group) + n = ssl.EC_GROUP_get_degree(group) + e = ssl.BN_CTX_get(ctx) + ssl.BN_bin2bn(msg, msglen, e) + if 8 * msglen > n: ssl.BN_rshift(e, e, 8 - (n & 7)) + + zero = ssl.BN_CTX_get(ctx) + ssl.BN_set_word(zero, 0) + ssl.BN_mod_sub(e, zero, e, order, ctx) + rr = ssl.BN_CTX_get(ctx); + ssl.BN_mod_inverse(rr, r, order, ctx) + sor = ssl.BN_CTX_get(ctx) + ssl.BN_mod_mul(sor, s, rr, order, ctx) + eor = ssl.BN_CTX_get(ctx) + ssl.BN_mod_mul(eor, e, rr, order, ctx) + ssl.EC_POINT_mul(group, Q, eor, R, sor, ctx) + ssl.EC_KEY_set_public_key(eckey, Q) + return eckey + finally: + if ctx: ssl.BN_CTX_free(ctx) + if R: ssl.EC_POINT_free(R) + if O: ssl.EC_POINT_free(O) + if Q: ssl.EC_POINT_free(Q) + +def closeLibrary(): import _ctypes if "FreeLibrary" in dir(_ctypes): _ctypes.FreeLibrary(ssl._lib._handle) @@ -354,7 +373,12 @@ def getMessagePubkey(message, sig): size = ssl.i2o_ECPublicKey (eckey, 0) mb = ctypes.create_string_buffer (size) ssl.i2o_ECPublicKey (eckey, ctypes.byref (ctypes.pointer (mb))) - return mb.raw + pub = mb.raw + if time.time()-ssl.time_opened>60*5: # Reopen every 5 min + logging.debug("Reopening OpenSSL...") + closeLibrary() + openLibrary() + return pub def test(): sign = "HGbib2kv9gm9IJjDt1FXbXFczZi35u0rZR3iPUIt5GglDDCeIQ7v8eYXVNIaLoJRI4URGZrhwmsYQ9aVtRTnTfQ=" diff --git a/src/lib/opensslVerify/stablityTest.py b/src/lib/opensslVerify/stablityTest.py index d6b498c4..2296e1f0 100644 --- a/src/lib/opensslVerify/stablityTest.py +++ b/src/lib/opensslVerify/stablityTest.py @@ -1,16 +1,35 @@ import opensslVerify, gevent, time -from gevent import monkey; monkey.patch_all(thread=False, ssl=False) +from gevent import monkey +monkey.patch_all(thread=False, ssl=False) def test(): data = "A"*1024 sign = "G2Jo8dDa+jqvJipft9E3kfrAxjESWLBpVtuGIiEBCD/UUyHmRMYNqnlWeOiaHHpja5LOP+U5CanRALfOjCSYIa8=" - for i in range(5*1000): + for i in range(2*1000): if i%1000 == 0: print i, len(data) - data += data+"A" + #data += data+"A" time.sleep(0) pub = opensslVerify.getMessagePubkey(data, sign) print repr(pub), len(data) -gevent.joinall([gevent.spawn(test), gevent.spawn(test)]) \ No newline at end of file +while 1: + s = time.time() + gevent.joinall([gevent.spawn(test), gevent.spawn(test)]) + try: + import psutil, os + process = psutil.Process(os.getpid()) + print "Mem:", process.get_memory_info()[0] / float(2 ** 20) + except: + pass + raw_input("finished, in %.2fs, check memory usage" % (time.time()-s)) + opensslVerify.close() + opensslVerify.open() + try: + import psutil, os + process = psutil.Process(os.getpid()) + print "Mem:", process.get_memory_info()[0] / float(2 ** 20) + except: + pass + raw_input("closed and openssl, check memory again, press enter to start again") diff --git a/zeronet.py b/zeronet.py index 71ce0764..8d1e46d9 100644 --- a/zeronet.py +++ b/zeronet.py @@ -13,7 +13,7 @@ def main(): # Try cleanup openssl try: if "lib.opensslVerify" in sys.modules: - sys.modules["lib.opensslVerify"].opensslVerify.close() + sys.modules["lib.opensslVerify"].opensslVerify.closeLibrary() except Exception, err: print "Error closing openssl", err From 66eca389bfc87049c1e20a26a5c0c2b8b876975e Mon Sep 17 00:00:00 2001 From: HelloZeroNet Date: Sun, 31 May 2015 15:52:21 +0200 Subject: [PATCH 2/2] rev 196, Configurable data dir, log dir and config file, fix for getFile unhandled exceptions, add http://zeronet.io to readme --- README.md | 2 +- plugins/Stats/StatsPlugin.py | 22 ++++++++-------- .../disabled-Dnschain/SiteManagerPlugin.py | 2 +- src/Config.py | 22 ++++++++++------ src/Debug/DebugReloader.py | 2 +- src/File/FileRequest.py | 2 +- src/Site/Site.py | 6 ++--- src/Site/SiteManager.py | 5 ++-- src/Site/SiteStorage.py | 3 ++- src/Test/test.py | 12 ++++----- src/Ui/UiRequest.py | 4 +-- src/User/User.py | 6 +++-- src/User/UserManager.py | 3 ++- src/Worker/Worker.py | 6 ++++- src/main.py | 25 ++++++++++--------- 15 files changed, 69 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 5c54512f..5ea16c37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ZeroNet -Decentralized websites using Bitcoin crypto and the BitTorrent network +Decentralized websites using Bitcoin crypto and the BitTorrent network - http://zeronet.io ## Why? diff --git a/plugins/Stats/StatsPlugin.py b/plugins/Stats/StatsPlugin.py index a6b43475..9e0c1cbf 100644 --- a/plugins/Stats/StatsPlugin.py +++ b/plugins/Stats/StatsPlugin.py @@ -394,7 +394,7 @@ class UiRequestPlugin(object): schema = { "db_name": "TestDb", - "db_file": "data/benchmark.db", + "db_file": "%s/benchmark.db" % config.data_dir, "maps": { ".*": { "to_table": { @@ -415,17 +415,17 @@ class UiRequestPlugin(object): } } - if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") + if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) with benchmark("Open x 10", 0.13): for i in range(10): - db = Db(schema, "data/benchmark.db") + db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() db.close() yield "." - db = Db(schema, "data/benchmark.db") + db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() import json @@ -434,9 +434,9 @@ class UiRequestPlugin(object): data = {"test": []} for i in range(1000): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) - json.dump(data, open("data/test_%s.json" % u, "w")) - db.loadJson("data/test_%s.json" % u) - os.unlink("data/test_%s.json" % u) + json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) + db.loadJson("%s/test_%s.json" % (config.data_dir, u)) + os.unlink("%s/test_%s.json" % (config.data_dir, u)) yield "." @@ -448,9 +448,9 @@ class UiRequestPlugin(object): data = {"test": []} for i in range(100): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) - json.dump(data, open("data/test_%s.json" % u, "w")) - db.loadJson("data/test_%s.json" % u, cur=cur) - os.unlink("data/test_%s.json" % u) + json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) + db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur) + os.unlink("%s/test_%s.json" % (config.data_dir, u)) if u%10 == 0: yield "." cur.execute("COMMIT") @@ -496,7 +496,7 @@ class UiRequestPlugin(object): db.close() - if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") + if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) gc.collect() # Implicit grabage collection diff --git a/plugins/disabled-Dnschain/SiteManagerPlugin.py b/plugins/disabled-Dnschain/SiteManagerPlugin.py index 6f757e8d..9121b425 100644 --- a/plugins/disabled-Dnschain/SiteManagerPlugin.py +++ b/plugins/disabled-Dnschain/SiteManagerPlugin.py @@ -11,7 +11,7 @@ log = logging.getLogger("DnschainPlugin") @PluginManager.registerTo("SiteManager") class SiteManagerPlugin(object): - dns_cache_path = "data/dns_cache.json" + dns_cache_path = "%s/dns_cache.json" % config.data_dir dns_cache = None # Checks if its a valid address diff --git a/src/Config.py b/src/Config.py index 834e1630..bd6f245d 100644 --- a/src/Config.py +++ b/src/Config.py @@ -4,7 +4,7 @@ import ConfigParser class Config(object): def __init__(self): self.version = "0.3.0" - self.rev = 194 + self.rev = 196 self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file @@ -103,12 +103,16 @@ class Config(object): parser.add_argument('--debug', help='Debug mode', action='store_true') parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true') + parser.add_argument('--config_file', help='Path of config file', default="zeronet.conf", metavar="path") + parser.add_argument('--data_dir', help='Path of data directory', default="data", metavar="path") + parser.add_argument('--log_dir', help='Path of logging directory', default="log", metavar="path") + parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port') parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*') parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name') parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address') - parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit') + parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size') parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port') @@ -151,17 +155,19 @@ class Config(object): action = self.getAction(argv) if len(argv) == 1 or not action: # If no action specificed set the main action argv.append("main") - if "zeronet.py" in argv[0]: - self.arguments = self.parser.parse_args(argv[1:]) - else: # Silent errors if not started with zeronet.py - self.arguments = self.parser.parse_args(argv[1:]) + self.arguments = self.parser.parse_args(argv[1:]) # Parse config file def parseConfig(self, argv): - if os.path.isfile("zeronet.conf"): + # Find config file path from parameters + config_file = "zeronet.conf" + if "--config_file" in argv: + config_file = argv[argv.index("--config_file")+1] + # Load config file + if os.path.isfile(config_file): config = ConfigParser.ConfigParser(allow_no_value=True) - config.read('zeronet.conf') + config.read(config_file) for section in config.sections(): for key, val in config.items(section): if section != "global": # If not global prefix key with section diff --git a/src/Debug/DebugReloader.py b/src/Debug/DebugReloader.py index d942600f..b3ae437b 100644 --- a/src/Debug/DebugReloader.py +++ b/src/Debug/DebugReloader.py @@ -35,7 +35,7 @@ class DebugReloader: def changed(self, evt): - if not evt.path or "data/" in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec + if not evt.path or "%s/" % config.data_dir in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec #logging.debug("Changed: %s" % evt) time.sleep(0.1) # Wait for lock release self.callback() diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index dfb2e8b0..2da28db0 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -154,7 +154,7 @@ class FileRequest(object): for peer in params["peers"]: # Add sent peers to site address = self.unpackAddress(peer) got_peer_keys.append("%s:%s" % address) - if (site.addPeer(*address)): added += 1 + if site.addPeer(*address): added += 1 # Send back peers that is not in the sent list and connectable (not port 0) packed_peers = [peer.packAddress() for peer in site.getConnectablePeers(params["need"], got_peer_keys)] if added: diff --git a/src/Site/Site.py b/src/Site/Site.py index 3dc81128..07acc786 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -59,7 +59,7 @@ class Site: # Load site settings from data/sites.json def loadSettings(self): - sites_settings = json.load(open("data/sites.json")) + sites_settings = json.load(open("%s/sites.json" % config.data_dir)) if self.address in sites_settings: self.settings = sites_settings[self.address] else: @@ -73,9 +73,9 @@ class Site: # Save site settings to data/sites.json def saveSettings(self): - sites_settings = json.load(open("data/sites.json")) + sites_settings = json.load(open("%s/sites.json" % config.data_dir)) sites_settings[self.address] = self.settings - open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True)) + open("%s/sites.json" % config.data_dir, "w").write(json.dumps(sites_settings, indent=2, sort_keys=True)) return diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 7a7e4d14..0d527109 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -1,6 +1,7 @@ import json, logging, time, re, os import gevent from Plugin import PluginManager +from Config import config TRACKERS = [ ("udp", "open.demonii.com", 1337), @@ -33,8 +34,8 @@ class SiteManager(object): address_found = [] added = 0 # Load new adresses - for address in json.load(open("data/sites.json")): - if address not in self.sites and os.path.isfile("data/%s/content.json" % address): + for address in json.load(open("%s/sites.json" % config.data_dir)): + if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)): self.sites[address] = Site(address) added += 1 address_found.append(address) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index fd08ae85..e41602c7 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -2,12 +2,13 @@ import os, re, shutil, json, time, sqlite3 import gevent.event from Db import Db from Debug import Debug +from Config import config class SiteStorage: def __init__(self, site, allow_create=True): self.site = site - self.directory = "data/%s" % self.site.address # Site data diretory + self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory self.log = site.log self.db = None # Db class self.db_checked = False # Checked db tables since startup diff --git a/src/Test/test.py b/src/Test/test.py index 250d015d..b1455189 100644 --- a/src/Test/test.py +++ b/src/Test/test.py @@ -120,11 +120,11 @@ class TestCase(unittest.TestCase): def testDb(self): print "Importing db..." from Db import Db - for db_path in [os.path.abspath("data/test/zeronet.db"), "data/test/zeronet.db"]: + for db_path in [os.path.abspath("%s/test/zeronet.db" % config.data_dir), "%s/test/zeronet.db" % config.data_dir]: print "Creating db using %s..." % db_path, schema = { "db_name": "TestDb", - "db_file": "data/test/zeronet.db", + "db_file": "%s/test/zeronet.db" % config.data_dir, "map": { "data.json": { "to_table": { @@ -144,14 +144,14 @@ class TestCase(unittest.TestCase): } } - if os.path.isfile("data/test/zeronet.db"): os.unlink("data/test/zeronet.db") - db = Db(schema, "data/test/zeronet.db") + if os.path.isfile("%s/test/zeronet.db" % config.data_dir): os.unlink("%s/test/zeronet.db" % config.data_dir) + db = Db(schema, "%s/test/zeronet.db" % config.data_dir) db.checkTables() db.close() # Cleanup - os.unlink("data/test/zeronet.db") - os.rmdir("data/test/") + os.unlink("%s/test/zeronet.db" % config.data_dir) + os.rmdir("%s/test/" % config.data_dir) def testContentManagerIncludes(self): diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 6fb7bd67..3a4a083a 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -236,8 +236,8 @@ class UiRequest(object): if match: # Looks like a valid path address = match.group("address") - file_path = "data/%s/%s" % (address, match.group("inner_path")) - allowed_dir = os.path.abspath("data/%s" % address) # Only files within data/sitehash allowed + file_path = "%s/%s/%s" % (config.data_dir, address, match.group("inner_path")) + allowed_dir = os.path.abspath("%s/%s" % (config.data_dir, address)) # Only files within data/sitehash allowed data_dir = os.path.abspath("data") # No files from data/ allowed if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir) or allowed_dir == data_dir: # File not in allowed path return self.error403() diff --git a/src/User/User.py b/src/User/User.py index 1d9f3a0a..57e89f68 100644 --- a/src/User/User.py +++ b/src/User/User.py @@ -1,6 +1,8 @@ import logging, json, time from Crypt import CryptBitcoin from Plugin import PluginManager +from Config import config + @PluginManager.acceptPlugins class User(object): @@ -22,13 +24,13 @@ class User(object): # Save to data/users.json def save(self): - users = json.load(open("data/users.json")) + users = json.load(open("%s/users.json" % config.data_dir)) if not self.master_address in users: users[self.master_address] = {} # Create if not exits user_data = users[self.master_address] if self.master_seed: user_data["master_seed"] = self.master_seed user_data["sites"] = self.sites user_data["certs"] = self.certs - open("data/users.json", "w").write(json.dumps(users, indent=2, sort_keys=True)) + open("%s/users.json" % config.data_dir, "w").write(json.dumps(users, indent=2, sort_keys=True)) self.log.debug("Saved") diff --git a/src/User/UserManager.py b/src/User/UserManager.py index 4b16b121..ed7b36e3 100644 --- a/src/User/UserManager.py +++ b/src/User/UserManager.py @@ -1,6 +1,7 @@ import json, logging, os from User import User from Plugin import PluginManager +from Config import config @PluginManager.acceptPlugins @@ -16,7 +17,7 @@ class UserManager(object): user_found = [] added = 0 # Load new users - for master_address, data in json.load(open("data/users.json")).items(): + for master_address, data in json.load(open("%s/users.json" % config.data_dir)).items(): if master_address not in self.users: user = User(master_address, data=data) self.users[master_address] = user diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py index 2f5d4c2b..96ff7801 100644 --- a/src/Worker/Worker.py +++ b/src/Worker/Worker.py @@ -40,7 +40,11 @@ class Worker: self.task = task site = task["site"] task["workers_num"] += 1 - buff = self.peer.getFile(site.address, task["inner_path"]) + try: + buff = self.peer.getFile(site.address, task["inner_path"]) + except Exception, err: + self.manager.log.debug("%s: getFile error: err" % (self.key, err)) + buff = None if self.running == False: # Worker no longer needed or got killed self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"])) break diff --git a/src/main.py b/src/main.py index 81fe96f0..86bce8aa 100644 --- a/src/main.py +++ b/src/main.py @@ -1,22 +1,23 @@ import os, sys update_after_shutdown = False # If set True then update and restart zeronet after main loop ended -# Create necessary files and dirs -if not os.path.isdir("log"): os.mkdir("log") -if not os.path.isdir("data"): os.mkdir("data") -if not os.path.isfile("data/sites.json"): open("data/sites.json", "w").write("{}") -if not os.path.isfile("data/users.json"): open("data/users.json", "w").write("{}") - # Load config from Config import config +# Create necessary files and dirs +if not os.path.isdir(config.log_dir): os.mkdir(config.log_dir) +if not os.path.isdir(config.data_dir): os.mkdir(config.data_dir) +if not os.path.isfile("%s/sites.json" % config.data_dir): open("%s/sites.json" % config.data_dir, "w").write("{}") +if not os.path.isfile("%s/users.json" % config.data_dir): open("%s/users.json" % config.data_dir, "w").write("{}") + + # Setup logging import logging if config.action == "main": - if os.path.isfile("log/debug.log"): # Simple logrotate - if os.path.isfile("log/debug-last.log"): os.unlink("log/debug-last.log") - os.rename("log/debug.log", "log/debug-last.log") - logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="log/debug.log") + if os.path.isfile("%s/debug.log" % config.log_dir): # Simple logrotate + if os.path.isfile("%s/debug-last.log" % config.log_dir): os.unlink("%s/debug-last.log" % config.log_dir) + os.rename("%s/debug.log" % config.log_dir, "%s/debug-last.log" % config.log_dir) + logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="%s/debug.log" % config.log_dir) else: logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main @@ -99,8 +100,8 @@ class Actions: logging.info("Creating directory structure...") from Site import Site - os.mkdir("data/%s" % address) - open("data/%s/index.html" % address, "w").write("Hello %s!" % address) + os.mkdir("%s/%s" % (config.data_dir, address)) + open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address) logging.info("Creating content.json...") site = Site(address)