diff --git a/README.md b/README.md index 5c54512f..5ea16c37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ZeroNet -Decentralized websites using Bitcoin crypto and the BitTorrent network +Decentralized websites using Bitcoin crypto and the BitTorrent network - http://zeronet.io ## Why? diff --git a/plugins/Stats/StatsPlugin.py b/plugins/Stats/StatsPlugin.py index a6b43475..9e0c1cbf 100644 --- a/plugins/Stats/StatsPlugin.py +++ b/plugins/Stats/StatsPlugin.py @@ -394,7 +394,7 @@ class UiRequestPlugin(object): schema = { "db_name": "TestDb", - "db_file": "data/benchmark.db", + "db_file": "%s/benchmark.db" % config.data_dir, "maps": { ".*": { "to_table": { @@ -415,17 +415,17 @@ class UiRequestPlugin(object): } } - if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") + if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) with benchmark("Open x 10", 0.13): for i in range(10): - db = Db(schema, "data/benchmark.db") + db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() db.close() yield "." - db = Db(schema, "data/benchmark.db") + db = Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() import json @@ -434,9 +434,9 @@ class UiRequestPlugin(object): data = {"test": []} for i in range(1000): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) - json.dump(data, open("data/test_%s.json" % u, "w")) - db.loadJson("data/test_%s.json" % u) - os.unlink("data/test_%s.json" % u) + json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) + db.loadJson("%s/test_%s.json" % (config.data_dir, u)) + os.unlink("%s/test_%s.json" % (config.data_dir, u)) yield "." @@ -448,9 +448,9 @@ class UiRequestPlugin(object): data = {"test": []} for i in range(100): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) - json.dump(data, open("data/test_%s.json" % u, "w")) - db.loadJson("data/test_%s.json" % u, cur=cur) - os.unlink("data/test_%s.json" % u) + json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) + db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur) + os.unlink("%s/test_%s.json" % (config.data_dir, u)) if u%10 == 0: yield "." cur.execute("COMMIT") @@ -496,7 +496,7 @@ class UiRequestPlugin(object): db.close() - if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db") + if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) gc.collect() # Implicit grabage collection diff --git a/plugins/disabled-Dnschain/SiteManagerPlugin.py b/plugins/disabled-Dnschain/SiteManagerPlugin.py index 6f757e8d..9121b425 100644 --- a/plugins/disabled-Dnschain/SiteManagerPlugin.py +++ b/plugins/disabled-Dnschain/SiteManagerPlugin.py @@ -11,7 +11,7 @@ log = logging.getLogger("DnschainPlugin") @PluginManager.registerTo("SiteManager") class SiteManagerPlugin(object): - dns_cache_path = "data/dns_cache.json" + dns_cache_path = "%s/dns_cache.json" % config.data_dir dns_cache = None # Checks if its a valid address diff --git a/src/Config.py b/src/Config.py index 834e1630..bd6f245d 100644 --- a/src/Config.py +++ b/src/Config.py @@ -4,7 +4,7 @@ import ConfigParser class Config(object): def __init__(self): self.version = "0.3.0" - self.rev = 194 + self.rev = 196 self.parser = self.createArguments() argv = sys.argv[:] # Copy command line arguments argv = self.parseConfig(argv) # Add arguments from config file @@ -103,12 +103,16 @@ class Config(object): parser.add_argument('--debug', help='Debug mode', action='store_true') parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true') + parser.add_argument('--config_file', help='Path of config file', default="zeronet.conf", metavar="path") + parser.add_argument('--data_dir', help='Path of data directory', default="data", metavar="path") + parser.add_argument('--log_dir', help='Path of logging directory', default="log", metavar="path") + parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip') parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port') parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*') parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name') parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address') - parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit') + parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size') parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip') parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port') @@ -151,17 +155,19 @@ class Config(object): action = self.getAction(argv) if len(argv) == 1 or not action: # If no action specificed set the main action argv.append("main") - if "zeronet.py" in argv[0]: - self.arguments = self.parser.parse_args(argv[1:]) - else: # Silent errors if not started with zeronet.py - self.arguments = self.parser.parse_args(argv[1:]) + self.arguments = self.parser.parse_args(argv[1:]) # Parse config file def parseConfig(self, argv): - if os.path.isfile("zeronet.conf"): + # Find config file path from parameters + config_file = "zeronet.conf" + if "--config_file" in argv: + config_file = argv[argv.index("--config_file")+1] + # Load config file + if os.path.isfile(config_file): config = ConfigParser.ConfigParser(allow_no_value=True) - config.read('zeronet.conf') + config.read(config_file) for section in config.sections(): for key, val in config.items(section): if section != "global": # If not global prefix key with section diff --git a/src/Debug/DebugReloader.py b/src/Debug/DebugReloader.py index d942600f..b3ae437b 100644 --- a/src/Debug/DebugReloader.py +++ b/src/Debug/DebugReloader.py @@ -35,7 +35,7 @@ class DebugReloader: def changed(self, evt): - if not evt.path or "data/" in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec + if not evt.path or "%s/" % config.data_dir in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec #logging.debug("Changed: %s" % evt) time.sleep(0.1) # Wait for lock release self.callback() diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index dfb2e8b0..2da28db0 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -154,7 +154,7 @@ class FileRequest(object): for peer in params["peers"]: # Add sent peers to site address = self.unpackAddress(peer) got_peer_keys.append("%s:%s" % address) - if (site.addPeer(*address)): added += 1 + if site.addPeer(*address): added += 1 # Send back peers that is not in the sent list and connectable (not port 0) packed_peers = [peer.packAddress() for peer in site.getConnectablePeers(params["need"], got_peer_keys)] if added: diff --git a/src/Site/Site.py b/src/Site/Site.py index 3dc81128..07acc786 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -59,7 +59,7 @@ class Site: # Load site settings from data/sites.json def loadSettings(self): - sites_settings = json.load(open("data/sites.json")) + sites_settings = json.load(open("%s/sites.json" % config.data_dir)) if self.address in sites_settings: self.settings = sites_settings[self.address] else: @@ -73,9 +73,9 @@ class Site: # Save site settings to data/sites.json def saveSettings(self): - sites_settings = json.load(open("data/sites.json")) + sites_settings = json.load(open("%s/sites.json" % config.data_dir)) sites_settings[self.address] = self.settings - open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True)) + open("%s/sites.json" % config.data_dir, "w").write(json.dumps(sites_settings, indent=2, sort_keys=True)) return diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 7a7e4d14..0d527109 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -1,6 +1,7 @@ import json, logging, time, re, os import gevent from Plugin import PluginManager +from Config import config TRACKERS = [ ("udp", "open.demonii.com", 1337), @@ -33,8 +34,8 @@ class SiteManager(object): address_found = [] added = 0 # Load new adresses - for address in json.load(open("data/sites.json")): - if address not in self.sites and os.path.isfile("data/%s/content.json" % address): + for address in json.load(open("%s/sites.json" % config.data_dir)): + if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)): self.sites[address] = Site(address) added += 1 address_found.append(address) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index fd08ae85..e41602c7 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -2,12 +2,13 @@ import os, re, shutil, json, time, sqlite3 import gevent.event from Db import Db from Debug import Debug +from Config import config class SiteStorage: def __init__(self, site, allow_create=True): self.site = site - self.directory = "data/%s" % self.site.address # Site data diretory + self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory self.log = site.log self.db = None # Db class self.db_checked = False # Checked db tables since startup diff --git a/src/Test/test.py b/src/Test/test.py index 250d015d..b1455189 100644 --- a/src/Test/test.py +++ b/src/Test/test.py @@ -120,11 +120,11 @@ class TestCase(unittest.TestCase): def testDb(self): print "Importing db..." from Db import Db - for db_path in [os.path.abspath("data/test/zeronet.db"), "data/test/zeronet.db"]: + for db_path in [os.path.abspath("%s/test/zeronet.db" % config.data_dir), "%s/test/zeronet.db" % config.data_dir]: print "Creating db using %s..." % db_path, schema = { "db_name": "TestDb", - "db_file": "data/test/zeronet.db", + "db_file": "%s/test/zeronet.db" % config.data_dir, "map": { "data.json": { "to_table": { @@ -144,14 +144,14 @@ class TestCase(unittest.TestCase): } } - if os.path.isfile("data/test/zeronet.db"): os.unlink("data/test/zeronet.db") - db = Db(schema, "data/test/zeronet.db") + if os.path.isfile("%s/test/zeronet.db" % config.data_dir): os.unlink("%s/test/zeronet.db" % config.data_dir) + db = Db(schema, "%s/test/zeronet.db" % config.data_dir) db.checkTables() db.close() # Cleanup - os.unlink("data/test/zeronet.db") - os.rmdir("data/test/") + os.unlink("%s/test/zeronet.db" % config.data_dir) + os.rmdir("%s/test/" % config.data_dir) def testContentManagerIncludes(self): diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 6fb7bd67..3a4a083a 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -236,8 +236,8 @@ class UiRequest(object): if match: # Looks like a valid path address = match.group("address") - file_path = "data/%s/%s" % (address, match.group("inner_path")) - allowed_dir = os.path.abspath("data/%s" % address) # Only files within data/sitehash allowed + file_path = "%s/%s/%s" % (config.data_dir, address, match.group("inner_path")) + allowed_dir = os.path.abspath("%s/%s" % (config.data_dir, address)) # Only files within data/sitehash allowed data_dir = os.path.abspath("data") # No files from data/ allowed if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir) or allowed_dir == data_dir: # File not in allowed path return self.error403() diff --git a/src/User/User.py b/src/User/User.py index 1d9f3a0a..57e89f68 100644 --- a/src/User/User.py +++ b/src/User/User.py @@ -1,6 +1,8 @@ import logging, json, time from Crypt import CryptBitcoin from Plugin import PluginManager +from Config import config + @PluginManager.acceptPlugins class User(object): @@ -22,13 +24,13 @@ class User(object): # Save to data/users.json def save(self): - users = json.load(open("data/users.json")) + users = json.load(open("%s/users.json" % config.data_dir)) if not self.master_address in users: users[self.master_address] = {} # Create if not exits user_data = users[self.master_address] if self.master_seed: user_data["master_seed"] = self.master_seed user_data["sites"] = self.sites user_data["certs"] = self.certs - open("data/users.json", "w").write(json.dumps(users, indent=2, sort_keys=True)) + open("%s/users.json" % config.data_dir, "w").write(json.dumps(users, indent=2, sort_keys=True)) self.log.debug("Saved") diff --git a/src/User/UserManager.py b/src/User/UserManager.py index 4b16b121..ed7b36e3 100644 --- a/src/User/UserManager.py +++ b/src/User/UserManager.py @@ -1,6 +1,7 @@ import json, logging, os from User import User from Plugin import PluginManager +from Config import config @PluginManager.acceptPlugins @@ -16,7 +17,7 @@ class UserManager(object): user_found = [] added = 0 # Load new users - for master_address, data in json.load(open("data/users.json")).items(): + for master_address, data in json.load(open("%s/users.json" % config.data_dir)).items(): if master_address not in self.users: user = User(master_address, data=data) self.users[master_address] = user diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py index 2f5d4c2b..96ff7801 100644 --- a/src/Worker/Worker.py +++ b/src/Worker/Worker.py @@ -40,7 +40,11 @@ class Worker: self.task = task site = task["site"] task["workers_num"] += 1 - buff = self.peer.getFile(site.address, task["inner_path"]) + try: + buff = self.peer.getFile(site.address, task["inner_path"]) + except Exception, err: + self.manager.log.debug("%s: getFile error: err" % (self.key, err)) + buff = None if self.running == False: # Worker no longer needed or got killed self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"])) break diff --git a/src/main.py b/src/main.py index 81fe96f0..86bce8aa 100644 --- a/src/main.py +++ b/src/main.py @@ -1,22 +1,23 @@ import os, sys update_after_shutdown = False # If set True then update and restart zeronet after main loop ended -# Create necessary files and dirs -if not os.path.isdir("log"): os.mkdir("log") -if not os.path.isdir("data"): os.mkdir("data") -if not os.path.isfile("data/sites.json"): open("data/sites.json", "w").write("{}") -if not os.path.isfile("data/users.json"): open("data/users.json", "w").write("{}") - # Load config from Config import config +# Create necessary files and dirs +if not os.path.isdir(config.log_dir): os.mkdir(config.log_dir) +if not os.path.isdir(config.data_dir): os.mkdir(config.data_dir) +if not os.path.isfile("%s/sites.json" % config.data_dir): open("%s/sites.json" % config.data_dir, "w").write("{}") +if not os.path.isfile("%s/users.json" % config.data_dir): open("%s/users.json" % config.data_dir, "w").write("{}") + + # Setup logging import logging if config.action == "main": - if os.path.isfile("log/debug.log"): # Simple logrotate - if os.path.isfile("log/debug-last.log"): os.unlink("log/debug-last.log") - os.rename("log/debug.log", "log/debug-last.log") - logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="log/debug.log") + if os.path.isfile("%s/debug.log" % config.log_dir): # Simple logrotate + if os.path.isfile("%s/debug-last.log" % config.log_dir): os.unlink("%s/debug-last.log" % config.log_dir) + os.rename("%s/debug.log" % config.log_dir, "%s/debug-last.log" % config.log_dir) + logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="%s/debug.log" % config.log_dir) else: logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main @@ -99,8 +100,8 @@ class Actions: logging.info("Creating directory structure...") from Site import Site - os.mkdir("data/%s" % address) - open("data/%s/index.html" % address, "w").write("Hello %s!" % address) + os.mkdir("%s/%s" % (config.data_dir, address)) + open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address) logging.info("Creating content.json...") site = Site(address)