rev 196, Configurable data dir, log dir and config file, fix for getFile unhandled exceptions, add http://zeronet.io to readme

This commit is contained in:
HelloZeroNet 2015-05-31 15:52:21 +02:00
parent 1c1d9fb75c
commit 66eca389bf
15 changed files with 69 additions and 53 deletions

View File

@ -1,6 +1,6 @@
# ZeroNet
Decentralized websites using Bitcoin crypto and the BitTorrent network
Decentralized websites using Bitcoin crypto and the BitTorrent network - http://zeronet.io
## Why?

View File

@ -394,7 +394,7 @@ class UiRequestPlugin(object):
schema = {
"db_name": "TestDb",
"db_file": "data/benchmark.db",
"db_file": "%s/benchmark.db" % config.data_dir,
"maps": {
".*": {
"to_table": {
@ -415,17 +415,17 @@ class UiRequestPlugin(object):
}
}
if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db")
if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir)
with benchmark("Open x 10", 0.13):
for i in range(10):
db = Db(schema, "data/benchmark.db")
db = Db(schema, "%s/benchmark.db" % config.data_dir)
db.checkTables()
db.close()
yield "."
db = Db(schema, "data/benchmark.db")
db = Db(schema, "%s/benchmark.db" % config.data_dir)
db.checkTables()
import json
@ -434,9 +434,9 @@ class UiRequestPlugin(object):
data = {"test": []}
for i in range(1000): # 1000 line of data
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
json.dump(data, open("data/test_%s.json" % u, "w"))
db.loadJson("data/test_%s.json" % u)
os.unlink("data/test_%s.json" % u)
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
db.loadJson("%s/test_%s.json" % (config.data_dir, u))
os.unlink("%s/test_%s.json" % (config.data_dir, u))
yield "."
@ -448,9 +448,9 @@ class UiRequestPlugin(object):
data = {"test": []}
for i in range(100): # 1000 line of data
data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)})
json.dump(data, open("data/test_%s.json" % u, "w"))
db.loadJson("data/test_%s.json" % u, cur=cur)
os.unlink("data/test_%s.json" % u)
json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w"))
db.loadJson("%s/test_%s.json" % (config.data_dir, u), cur=cur)
os.unlink("%s/test_%s.json" % (config.data_dir, u))
if u%10 == 0: yield "."
cur.execute("COMMIT")
@ -496,7 +496,7 @@ class UiRequestPlugin(object):
db.close()
if os.path.isfile("data/benchmark.db"): os.unlink("data/benchmark.db")
if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir)
gc.collect() # Implicit grabage collection

View File

@ -11,7 +11,7 @@ log = logging.getLogger("DnschainPlugin")
@PluginManager.registerTo("SiteManager")
class SiteManagerPlugin(object):
dns_cache_path = "data/dns_cache.json"
dns_cache_path = "%s/dns_cache.json" % config.data_dir
dns_cache = None
# Checks if its a valid address

View File

@ -4,7 +4,7 @@ import ConfigParser
class Config(object):
def __init__(self):
self.version = "0.3.0"
self.rev = 194
self.rev = 196
self.parser = self.createArguments()
argv = sys.argv[:] # Copy command line arguments
argv = self.parseConfig(argv) # Add arguments from config file
@ -103,12 +103,16 @@ class Config(object):
parser.add_argument('--debug', help='Debug mode', action='store_true')
parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true')
parser.add_argument('--config_file', help='Path of config file', default="zeronet.conf", metavar="path")
parser.add_argument('--data_dir', help='Path of data directory', default="data", metavar="path")
parser.add_argument('--log_dir', help='Path of logging directory', default="log", metavar="path")
parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip')
parser.add_argument('--ui_port', help='Web interface bind port', default=43110, type=int, metavar='port')
parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip', nargs='*')
parser.add_argument('--open_browser', help='Open homepage in web browser automatically', nargs='?', const="default_browser", metavar='browser_name')
parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size_limit')
parser.add_argument('--size_limit', help='Default site size limit in MB', default=10, metavar='size')
parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip')
parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, type=int, metavar='port')
@ -151,17 +155,19 @@ class Config(object):
action = self.getAction(argv)
if len(argv) == 1 or not action: # If no action specificed set the main action
argv.append("main")
if "zeronet.py" in argv[0]:
self.arguments = self.parser.parse_args(argv[1:])
else: # Silent errors if not started with zeronet.py
self.arguments = self.parser.parse_args(argv[1:])
self.arguments = self.parser.parse_args(argv[1:])
# Parse config file
def parseConfig(self, argv):
if os.path.isfile("zeronet.conf"):
# Find config file path from parameters
config_file = "zeronet.conf"
if "--config_file" in argv:
config_file = argv[argv.index("--config_file")+1]
# Load config file
if os.path.isfile(config_file):
config = ConfigParser.ConfigParser(allow_no_value=True)
config.read('zeronet.conf')
config.read(config_file)
for section in config.sections():
for key, val in config.items(section):
if section != "global": # If not global prefix key with section

View File

@ -35,7 +35,7 @@ class DebugReloader:
def changed(self, evt):
if not evt.path or "data/" in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec
if not evt.path or "%s/" % config.data_dir in evt.path or evt.path.endswith("pyc") or time.time()-self.last_chaged < 1: return False # Ignore *.pyc changes and no reload within 1 sec
#logging.debug("Changed: %s" % evt)
time.sleep(0.1) # Wait for lock release
self.callback()

View File

@ -154,7 +154,7 @@ class FileRequest(object):
for peer in params["peers"]: # Add sent peers to site
address = self.unpackAddress(peer)
got_peer_keys.append("%s:%s" % address)
if (site.addPeer(*address)): added += 1
if site.addPeer(*address): added += 1
# Send back peers that is not in the sent list and connectable (not port 0)
packed_peers = [peer.packAddress() for peer in site.getConnectablePeers(params["need"], got_peer_keys)]
if added:

View File

@ -59,7 +59,7 @@ class Site:
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
@ -73,9 +73,9 @@ class Site:
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
sites_settings[self.address] = self.settings
open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
open("%s/sites.json" % config.data_dir, "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
return

View File

@ -1,6 +1,7 @@
import json, logging, time, re, os
import gevent
from Plugin import PluginManager
from Config import config
TRACKERS = [
("udp", "open.demonii.com", 1337),
@ -33,8 +34,8 @@ class SiteManager(object):
address_found = []
added = 0
# Load new adresses
for address in json.load(open("data/sites.json")):
if address not in self.sites and os.path.isfile("data/%s/content.json" % address):
for address in json.load(open("%s/sites.json" % config.data_dir)):
if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
self.sites[address] = Site(address)
added += 1
address_found.append(address)

View File

@ -2,12 +2,13 @@ import os, re, shutil, json, time, sqlite3
import gevent.event
from Db import Db
from Debug import Debug
from Config import config
class SiteStorage:
def __init__(self, site, allow_create=True):
self.site = site
self.directory = "data/%s" % self.site.address # Site data diretory
self.directory = "%s/%s" % (config.data_dir, self.site.address) # Site data diretory
self.log = site.log
self.db = None # Db class
self.db_checked = False # Checked db tables since startup

View File

@ -120,11 +120,11 @@ class TestCase(unittest.TestCase):
def testDb(self):
print "Importing db..."
from Db import Db
for db_path in [os.path.abspath("data/test/zeronet.db"), "data/test/zeronet.db"]:
for db_path in [os.path.abspath("%s/test/zeronet.db" % config.data_dir), "%s/test/zeronet.db" % config.data_dir]:
print "Creating db using %s..." % db_path,
schema = {
"db_name": "TestDb",
"db_file": "data/test/zeronet.db",
"db_file": "%s/test/zeronet.db" % config.data_dir,
"map": {
"data.json": {
"to_table": {
@ -144,14 +144,14 @@ class TestCase(unittest.TestCase):
}
}
if os.path.isfile("data/test/zeronet.db"): os.unlink("data/test/zeronet.db")
db = Db(schema, "data/test/zeronet.db")
if os.path.isfile("%s/test/zeronet.db" % config.data_dir): os.unlink("%s/test/zeronet.db" % config.data_dir)
db = Db(schema, "%s/test/zeronet.db" % config.data_dir)
db.checkTables()
db.close()
# Cleanup
os.unlink("data/test/zeronet.db")
os.rmdir("data/test/")
os.unlink("%s/test/zeronet.db" % config.data_dir)
os.rmdir("%s/test/" % config.data_dir)
def testContentManagerIncludes(self):

View File

@ -236,8 +236,8 @@ class UiRequest(object):
if match: # Looks like a valid path
address = match.group("address")
file_path = "data/%s/%s" % (address, match.group("inner_path"))
allowed_dir = os.path.abspath("data/%s" % address) # Only files within data/sitehash allowed
file_path = "%s/%s/%s" % (config.data_dir, address, match.group("inner_path"))
allowed_dir = os.path.abspath("%s/%s" % (config.data_dir, address)) # Only files within data/sitehash allowed
data_dir = os.path.abspath("data") # No files from data/ allowed
if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir) or allowed_dir == data_dir: # File not in allowed path
return self.error403()

View File

@ -1,6 +1,8 @@
import logging, json, time
from Crypt import CryptBitcoin
from Plugin import PluginManager
from Config import config
@PluginManager.acceptPlugins
class User(object):
@ -22,13 +24,13 @@ class User(object):
# Save to data/users.json
def save(self):
users = json.load(open("data/users.json"))
users = json.load(open("%s/users.json" % config.data_dir))
if not self.master_address in users: users[self.master_address] = {} # Create if not exits
user_data = users[self.master_address]
if self.master_seed: user_data["master_seed"] = self.master_seed
user_data["sites"] = self.sites
user_data["certs"] = self.certs
open("data/users.json", "w").write(json.dumps(users, indent=2, sort_keys=True))
open("%s/users.json" % config.data_dir, "w").write(json.dumps(users, indent=2, sort_keys=True))
self.log.debug("Saved")

View File

@ -1,6 +1,7 @@
import json, logging, os
from User import User
from Plugin import PluginManager
from Config import config
@PluginManager.acceptPlugins
@ -16,7 +17,7 @@ class UserManager(object):
user_found = []
added = 0
# Load new users
for master_address, data in json.load(open("data/users.json")).items():
for master_address, data in json.load(open("%s/users.json" % config.data_dir)).items():
if master_address not in self.users:
user = User(master_address, data=data)
self.users[master_address] = user

View File

@ -40,7 +40,11 @@ class Worker:
self.task = task
site = task["site"]
task["workers_num"] += 1
buff = self.peer.getFile(site.address, task["inner_path"])
try:
buff = self.peer.getFile(site.address, task["inner_path"])
except Exception, err:
self.manager.log.debug("%s: getFile error: err" % (self.key, err))
buff = None
if self.running == False: # Worker no longer needed or got killed
self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"]))
break

View File

@ -1,22 +1,23 @@
import os, sys
update_after_shutdown = False # If set True then update and restart zeronet after main loop ended
# Create necessary files and dirs
if not os.path.isdir("log"): os.mkdir("log")
if not os.path.isdir("data"): os.mkdir("data")
if not os.path.isfile("data/sites.json"): open("data/sites.json", "w").write("{}")
if not os.path.isfile("data/users.json"): open("data/users.json", "w").write("{}")
# Load config
from Config import config
# Create necessary files and dirs
if not os.path.isdir(config.log_dir): os.mkdir(config.log_dir)
if not os.path.isdir(config.data_dir): os.mkdir(config.data_dir)
if not os.path.isfile("%s/sites.json" % config.data_dir): open("%s/sites.json" % config.data_dir, "w").write("{}")
if not os.path.isfile("%s/users.json" % config.data_dir): open("%s/users.json" % config.data_dir, "w").write("{}")
# Setup logging
import logging
if config.action == "main":
if os.path.isfile("log/debug.log"): # Simple logrotate
if os.path.isfile("log/debug-last.log"): os.unlink("log/debug-last.log")
os.rename("log/debug.log", "log/debug-last.log")
logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="log/debug.log")
if os.path.isfile("%s/debug.log" % config.log_dir): # Simple logrotate
if os.path.isfile("%s/debug-last.log" % config.log_dir): os.unlink("%s/debug-last.log" % config.log_dir)
os.rename("%s/debug.log" % config.log_dir, "%s/debug-last.log" % config.log_dir)
logging.basicConfig(format='[%(asctime)s] %(levelname)-8s %(name)s %(message)s', level=logging.DEBUG, filename="%s/debug.log" % config.log_dir)
else:
logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main
@ -99,8 +100,8 @@ class Actions:
logging.info("Creating directory structure...")
from Site import Site
os.mkdir("data/%s" % address)
open("data/%s/index.html" % address, "w").write("Hello %s!" % address)
os.mkdir("%s/%s" % (config.data_dir, address))
open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address)
logging.info("Creating content.json...")
site = Site(address)