limitations and irc to readme, version 0.1.2, socket debugging option, Notify exceptions support, better error logging, retry on socket error, dont expose external ip to websocket api, kill workers if no task, log time to console
This commit is contained in:
HelloZeroNet
parent
185424b815
commit
b37e309eda
13
README.md
13
README.md
|
|
@ -41,6 +41,13 @@ Linux (Debian):
|
|||
- start using `python zeronet.py`
|
||||
|
||||
|
||||
## Current limitations
|
||||
- No torrent-like, file splitting big file support
|
||||
- Just as anonymous as the bittorrent
|
||||
- File transactions not compressed or encrypted yet
|
||||
- No private sites
|
||||
|
||||
|
||||
## How can I create a ZeroNet site?
|
||||
Shut down zeronet.py if you are running it already
|
||||
```
|
||||
|
|
@ -55,6 +62,7 @@ $ zeronet.py
|
|||
```
|
||||
Congratulations, you are done! Now anyone can access your site using http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
|
||||
|
||||
|
||||
## How can I modify a ZeroNet site?
|
||||
- Modify files located in data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 directory. After you done:
|
||||
```
|
||||
|
|
@ -72,10 +80,13 @@ Site:13DNDk..bhC2 Successfuly published to 3 peers
|
|||
```
|
||||
- That's it! You successfuly signed and published your modifications.
|
||||
|
||||
|
||||
## If you want to help keep this project alive
|
||||
|
||||
Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX
|
||||
|
||||
#### Thank you!
|
||||
|
||||
More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
|
||||
|
||||
More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
|
||||
Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import ConfigParser
|
|||
|
||||
class Config(object):
|
||||
def __init__(self):
|
||||
self.version = "0.1.1"
|
||||
self.version = "0.1.2"
|
||||
self.parser = self.createArguments()
|
||||
argv = sys.argv[:] # Copy command line arguments
|
||||
argv = self.parseConfig(argv) # Add arguments from config file
|
||||
|
|
@ -53,13 +53,14 @@ class Config(object):
|
|||
|
||||
# Config parameters
|
||||
parser.add_argument('--debug', help='Debug mode', action='store_true')
|
||||
parser.add_argument('--debug_socket', help='Debug socket connections', action='store_true')
|
||||
|
||||
parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='host')
|
||||
parser.add_argument('--ui_ip', help='Web interface bind address', default="127.0.0.1", metavar='ip')
|
||||
parser.add_argument('--ui_port', help='Web interface bind port', default=43110, metavar='port')
|
||||
parser.add_argument('--ui_restrict', help='Restrict web access', default=False, metavar='ip')
|
||||
parser.add_argument('--homepage', help='Web interface Homepage', default='1EU1tbG9oC1A8jz2ouVwGZyQ5asrNsE4Vr', metavar='address')
|
||||
|
||||
parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='host')
|
||||
parser.add_argument('--fileserver_ip', help='FileServer bind address', default="*", metavar='ip')
|
||||
parser.add_argument('--fileserver_port',help='FileServer bind port', default=15441, metavar='port')
|
||||
|
||||
parser.add_argument('--ip_external', help='External ip (tested on start if None)', metavar='ip')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
import sys, os, traceback
|
||||
|
||||
# Non fatal exception
|
||||
class Notify(Exception):
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
def __str__(self):
|
||||
return self.message
|
||||
|
||||
|
||||
def formatException(err=None):
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
if not err: err = exc_obj.message
|
||||
tb = []
|
||||
for frame in traceback.extract_tb(exc_tb):
|
||||
path, line, function, text = frame
|
||||
file = os.path.split(path)[1]
|
||||
tb.append("%s line %s" % (file, line))
|
||||
return "%s: %s in %s" % (exc_type.__name__, err, " > ".join(tb))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
print 1/0
|
||||
except Exception, err:
|
||||
print type(err).__name__
|
||||
print "1/0 error: %s" % formatException(err)
|
||||
|
||||
def loadJson():
|
||||
json.loads("Errr")
|
||||
|
||||
import json
|
||||
try:
|
||||
loadJson()
|
||||
except Exception, err:
|
||||
print err
|
||||
print "Json load error: %s" % formatException(err)
|
||||
loadJson()
|
||||
|
|
@ -3,14 +3,14 @@ import gevent, sys
|
|||
last_error = None
|
||||
def handleError(*args):
|
||||
global last_error
|
||||
if not args: # Get last error
|
||||
if not args: # Called explicitly
|
||||
args = sys.exc_info()
|
||||
silent = True
|
||||
else:
|
||||
silent = False
|
||||
print "Error catched", args
|
||||
last_error = args
|
||||
if not silent: sys.__excepthook__(*args)
|
||||
if not silent and args[0].__name__ != "Notify": sys.__excepthook__(*args)
|
||||
|
||||
OriginalGreenlet = gevent.Greenlet
|
||||
class ErrorhookedGreenlet(OriginalGreenlet):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import os, msgpack, shutil
|
||||
from Site import SiteManager
|
||||
from cStringIO import StringIO
|
||||
from Debug import Debug
|
||||
|
||||
FILE_BUFF = 1024*512
|
||||
|
||||
|
|
@ -87,7 +88,7 @@ class FileRequest:
|
|||
back["size"] = os.fstat(file.fileno()).st_size
|
||||
self.send(back)
|
||||
except Exception, err:
|
||||
self.send({"error": "File read error: %s" % err})
|
||||
self.send({"error": "File read error: %s" % Debug.formatException(err)})
|
||||
return False
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import zmq.green as zmq
|
|||
from Config import config
|
||||
from FileRequest import FileRequest
|
||||
from Site import SiteManager
|
||||
from Debug import Debug
|
||||
|
||||
|
||||
class FileServer:
|
||||
|
|
@ -53,7 +54,7 @@ class FileServer:
|
|||
else:
|
||||
upnpc_success = False
|
||||
except Exception, err:
|
||||
self.log.error("Upnpc run error: %s" % err)
|
||||
self.log.error("Upnpc run error: %s" % Debug.formatException(err))
|
||||
upnpc_success = False
|
||||
|
||||
if upnpc_success and self.testOpenport(port)["result"] == True:
|
||||
|
|
@ -73,7 +74,7 @@ class FileServer:
|
|||
message = re.match('.*<p style="padding-left:15px">(.*?)</p>', data, re.DOTALL).group(1)
|
||||
message = re.sub("<.*?>", "", message.replace("<br>", " ").replace(" ", " ")) # Strip http tags
|
||||
except Exception, err:
|
||||
message = "Error: %s" % err
|
||||
message = "Error: %s" % Debug.formatException(err)
|
||||
if "Error" in message:
|
||||
self.log.info("[BAD :(] Port closed: %s" % message)
|
||||
if port == self.port:
|
||||
|
|
@ -159,7 +160,7 @@ class FileServer:
|
|||
self.handleRequest(req)
|
||||
except Exception, err:
|
||||
self.log.error(err)
|
||||
self.socket.send(msgpack.packb({"error": "%s" % err}, use_bin_type=True))
|
||||
self.socket.send(msgpack.packb({"error": "%s" % Debug.formatException(err)}, use_bin_type=True))
|
||||
if config.debug: # Raise exception
|
||||
import sys
|
||||
sys.excepthook(*sys.exc_info())
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import os, logging, gevent, time, msgpack
|
|||
import zmq.green as zmq
|
||||
from cStringIO import StringIO
|
||||
from Config import config
|
||||
from Debug import Debug
|
||||
|
||||
context = zmq.Context()
|
||||
|
||||
|
|
@ -40,23 +41,29 @@ class Peer:
|
|||
# Send a command to peer
|
||||
def sendCmd(self, cmd, params = {}):
|
||||
if not self.socket: self.connect()
|
||||
self.log.debug("sendCmd: %s" % cmd)
|
||||
try:
|
||||
self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True))
|
||||
response = msgpack.unpackb(self.socket.recv())
|
||||
if "error" in response:
|
||||
self.log.debug("%s %s error: %s" % (cmd, params, response["error"]))
|
||||
for retry in range(1,5):
|
||||
if config.debug_socket: self.log.debug("sendCmd: %s" % cmd)
|
||||
try:
|
||||
self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True))
|
||||
if config.debug_socket: self.log.debug("Sent command: %s" % cmd)
|
||||
response = msgpack.unpackb(self.socket.recv())
|
||||
if config.debug_socket: self.log.debug("Got response to: %s" % cmd)
|
||||
if "error" in response:
|
||||
self.log.debug("%s error: %s" % (cmd, response["error"]))
|
||||
self.onConnectionError()
|
||||
else: # Successful request, reset connection error num
|
||||
self.connection_error = 0
|
||||
return response
|
||||
except Exception, err:
|
||||
self.onConnectionError()
|
||||
else: # Successful request, reset connection error num
|
||||
self.connection_error = 0
|
||||
return response
|
||||
except Exception, err:
|
||||
self.onConnectionError()
|
||||
self.log.error("%s" % err)
|
||||
self.socket.close()
|
||||
time.sleep(1)
|
||||
self.connect()
|
||||
return None
|
||||
self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry))
|
||||
self.socket.close()
|
||||
time.sleep(1*retry)
|
||||
self.connect()
|
||||
if type(err).__name__ == "Notify" and err.message == "Worker stopped": # Greenlet kill by worker
|
||||
self.log.debug("Peer worker got killed, aborting cmd: %s" % cmd)
|
||||
break
|
||||
return None # Failed after 4 retry
|
||||
|
||||
|
||||
# Get a file content from peer
|
||||
|
|
@ -97,7 +104,7 @@ class Peer:
|
|||
# On connection error
|
||||
def onConnectionError(self):
|
||||
self.connection_error += 1
|
||||
if self.connection_error > 5: # Dead peer
|
||||
if self.connection_error >= 5: # Dead peer
|
||||
self.remove()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from Config import config
|
|||
from Peer import Peer
|
||||
from Worker import WorkerManager
|
||||
from Crypt import CryptHash
|
||||
from Debug import Debug
|
||||
import SiteManager
|
||||
|
||||
class Site:
|
||||
|
|
@ -53,7 +54,7 @@ class Site:
|
|||
try:
|
||||
new_content = json.load(open(content_path))
|
||||
except Exception, err:
|
||||
self.log.error("Content.json load error: %s" % err)
|
||||
self.log.error("Content.json load error: %s" % Debug.formatException(err))
|
||||
return None
|
||||
else:
|
||||
return None # Content.json not exits
|
||||
|
|
@ -69,7 +70,7 @@ class Site:
|
|||
if old_sha1 != new_sha1: changed.append(inner_path)
|
||||
self.content = new_content
|
||||
except Exception, err:
|
||||
self.log.error("Content.json parse error: %s" % err)
|
||||
self.log.error("Content.json parse error: %s" % Debug.formatException(err))
|
||||
return None # Content.json parse error
|
||||
# Add to bad files
|
||||
if not init:
|
||||
|
|
@ -114,7 +115,7 @@ class Site:
|
|||
# Start downloading site
|
||||
@util.Noparallel(blocking=False)
|
||||
def download(self):
|
||||
self.log.debug("Start downloading...")
|
||||
self.log.debug("Start downloading...%s" % self.bad_files)
|
||||
self.announce()
|
||||
found = self.needFile("content.json", update=self.bad_files.get("content.json"))
|
||||
if not found: return False # Could not download content.json
|
||||
|
|
@ -165,7 +166,7 @@ class Site:
|
|||
"peer": (config.ip_external, config.fileserver_port)
|
||||
})
|
||||
except Exception, err:
|
||||
result = {"exception": err}
|
||||
result = {"exception": Debug.formatException(err)}
|
||||
|
||||
if result and "ok" in result:
|
||||
published += 1
|
||||
|
|
@ -231,24 +232,22 @@ class Site:
|
|||
tracker.poll_once()
|
||||
tracker.announce(info_hash=hashlib.sha1(self.address).hexdigest(), num_want=50)
|
||||
back = tracker.poll_once()
|
||||
except Exception, err:
|
||||
self.log.error("Tracker error: %s" % err)
|
||||
continue
|
||||
if back: # Tracker announce success
|
||||
peers = back["response"]["peers"]
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself)
|
||||
continue
|
||||
if self.addPeer(peer["addr"], peer["port"]): added += 1
|
||||
if added:
|
||||
self.worker_manager.onPeers()
|
||||
self.updateWebsocket(peers_added=added)
|
||||
self.log.debug("Found %s peers, new: %s" % (len(peers), added))
|
||||
break # Successful announcing, break the list
|
||||
else:
|
||||
self.log.error("Tracker bad response, trying next in list...") # Failed to announce, go to next
|
||||
except Exception, err:
|
||||
self.log.error("Tracker error: %s" % Debug.formatException(err))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
added = 0
|
||||
for peer in peers:
|
||||
if (peer["addr"], peer["port"]) in self.peer_blacklist: # Ignore blacklist (eg. myself)
|
||||
continue
|
||||
if self.addPeer(peer["addr"], peer["port"]): added += 1
|
||||
if added:
|
||||
self.worker_manager.onPeers()
|
||||
self.updateWebsocket(peers_added=added)
|
||||
self.log.debug("Found %s peers, new: %s" % (len(peers), added))
|
||||
break # Successful announcing, break the list
|
||||
else:
|
||||
pass # TODO: http tracker support
|
||||
|
||||
|
|
@ -342,7 +341,7 @@ class Site:
|
|||
|
||||
return CryptBitcoin.verify(sign_content, self.address, sign)
|
||||
except Exception, err:
|
||||
self.log.error("Verify sign error: %s" % err)
|
||||
self.log.error("Verify sign error: %s" % Debug.formatException(err))
|
||||
return False
|
||||
|
||||
else: # Check using sha1 hash
|
||||
|
|
|
|||
|
|
@ -245,6 +245,7 @@ class UiRequest:
|
|||
|
||||
# Just raise an error to get console
|
||||
def actionConsole(self):
|
||||
sites = self.server.sites
|
||||
raise Exception("Here is your console")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from lib.geventwebsocket.handler import WebSocketHandler
|
|||
from Ui import UiRequest
|
||||
from Site import SiteManager
|
||||
from Config import config
|
||||
from Debug import Debug
|
||||
|
||||
# Skip websocket handler if not necessary
|
||||
class UiWSGIHandler(WSGIHandler):
|
||||
|
|
@ -48,19 +49,6 @@ class UiServer:
|
|||
return self.ui_request.route(path)
|
||||
|
||||
|
||||
# Send a message to all connected client
|
||||
def sendMessage(self, message):
|
||||
sent = 0
|
||||
for ws in self.websockets:
|
||||
try:
|
||||
ws.send(message)
|
||||
sent += 1
|
||||
except Exception, err:
|
||||
self.log.error("addMessage error: %s" % err)
|
||||
self.server.websockets.remove(ws)
|
||||
return sent
|
||||
|
||||
|
||||
# Reload the UiRequest class to prevent restarts in debug mode
|
||||
def reload(self):
|
||||
import imp
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import json, gevent, time, sys, hashlib
|
||||
from Config import config
|
||||
from Site import SiteManager
|
||||
from Debug import Debug
|
||||
|
||||
class UiWebsocket:
|
||||
def __init__(self, ws, site, server):
|
||||
|
|
@ -36,7 +37,7 @@ class UiWebsocket:
|
|||
if config.debug: # Allow websocket errors to appear on /Debug
|
||||
import sys
|
||||
sys.modules["src.main"].DebugHook.handleError()
|
||||
self.log.error("WebSocket error: %s" % err)
|
||||
self.log.error("WebSocket error: %s" % Debug.formatException(err))
|
||||
return "Bye."
|
||||
|
||||
|
||||
|
|
@ -70,7 +71,7 @@ class UiWebsocket:
|
|||
if cb: # Callback after client responsed
|
||||
self.waiting_cb[message["id"]] = cb
|
||||
except Exception, err:
|
||||
self.log.debug("Websocket send error: %s" % err)
|
||||
self.log.debug("Websocket send error: %s" % Debug.formatException(err))
|
||||
|
||||
|
||||
# Handle incoming messages
|
||||
|
|
@ -152,7 +153,7 @@ class UiWebsocket:
|
|||
# Server variables
|
||||
def actionServerInfo(self, to, params):
|
||||
ret = {
|
||||
"ip_external": config.ip_external,
|
||||
"ip_external": bool(config.ip_external),
|
||||
"platform": sys.platform,
|
||||
"fileserver_ip": config.fileserver_ip,
|
||||
"fileserver_port": config.fileserver_port,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import gevent, time, logging, shutil, os
|
||||
from Peer import Peer
|
||||
from Debug import Debug
|
||||
|
||||
class Worker:
|
||||
def __init__(self, manager, peer):
|
||||
|
|
@ -66,6 +67,11 @@ class Worker:
|
|||
self.running = True
|
||||
self.thread = gevent.spawn(self.downloader)
|
||||
|
||||
|
||||
# Force stop the worker
|
||||
def stop(self):
|
||||
self.manager.log.debug("%s: Force stopping, thread: %s" % (self.key, self.thread))
|
||||
self.running = False
|
||||
if self.thread:
|
||||
self.thread.kill(exception=Debug.Notify("Worker stopped"))
|
||||
self.manager.removeWorker(self)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,12 @@ class WorkerManager:
|
|||
# Check expired tasks
|
||||
def checkTasks(self):
|
||||
while 1:
|
||||
time.sleep(15) # Check every 30 sec
|
||||
time.sleep(15) # Check every 15 sec
|
||||
|
||||
# Clean up workers
|
||||
if not self.tasks and self.workers: # No task but workers still running
|
||||
for worker in self.workers.values(): worker.stop()
|
||||
|
||||
if not self.tasks: continue
|
||||
tasks = self.tasks[:] # Copy it so removing elements wont cause any problem
|
||||
for task in tasks:
|
||||
|
|
@ -40,6 +45,7 @@ class WorkerManager:
|
|||
|
||||
|
||||
|
||||
|
||||
# Tasks sorted by this
|
||||
def taskSorter(self, task):
|
||||
if task["inner_path"] == "content.json": return 9999 # Content.json always prority
|
||||
|
|
@ -96,8 +102,9 @@ class WorkerManager:
|
|||
# Ends and remove a worker
|
||||
def removeWorker(self, worker):
|
||||
worker.running = False
|
||||
if worker.key in self.workers: del(self.workers[worker.key])
|
||||
self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS))
|
||||
if worker.key in self.workers:
|
||||
del(self.workers[worker.key])
|
||||
self.log.debug("Removed worker, workers: %s/%s" % (len(self.workers), MAX_WORKERS))
|
||||
|
||||
|
||||
# Create new task and return asyncresult
|
||||
|
|
|
|||
|
|
@ -19,8 +19,14 @@ if config.action == "main":
|
|||
else:
|
||||
logging.basicConfig(level=logging.DEBUG, stream=open(os.devnull,"w")) # No file logging if action is not main
|
||||
|
||||
# Console logger
|
||||
console_log = logging.StreamHandler()
|
||||
console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S"))
|
||||
if config.action == "main": # Add time if main action
|
||||
console_log.setFormatter(logging.Formatter('[%(asctime)s] %(name)s %(message)s', "%H:%M:%S"))
|
||||
else:
|
||||
console_log.setFormatter(logging.Formatter('%(name)s %(message)s', "%H:%M:%S"))
|
||||
|
||||
|
||||
logging.getLogger('').addHandler(console_log) # Add console logger
|
||||
logging.getLogger('').name = "-" # Remove root prefix
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue