diff --git a/README.md b/README.md
index b2ac7429..86d25d36 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ Decentralized websites using Bitcoin crypto and BitTorrent network
- When you visit a new zeronet site, it's trying to find peers using BitTorrent network and download the site files (html, css, js...) from them.
- Each visited sites become also served by You.
- Every site containing a `site.json` which holds all other files sha1 hash and a sign generated using site's private key.
- - If the site owner (who has the private key for the site address) modifies the site, then he/she signs the new `content.json` and publish it to the peers. After the peers have verified the `content.json` integrity using the sign they download the modified files and publish the new content to other peers.
+ - If the site owner (who has the private key for the site address) modifies the site, then he/she signs the new `content.json` and publish it to the peers. After the peers have verified the `content.json` integrity (using the sign), they download the modified files and publish the new content to other peers.
## Screenshot
@@ -62,6 +62,8 @@ $ zeronet.py
```
Congratulations, you are done! Now anyone can access your site using http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
+Next steps: [ZeroNet Developer Documentation](https://github.com/HelloZeroNet/ZeroNet/wiki/ZeroNet-Developer-Documentation)
+
## How can I modify a ZeroNet site?
- Modify files located in data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 directory. After you done:
@@ -85,8 +87,8 @@ Site:13DNDk..bhC2 Successfuly published to 3 peers
Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX
+
#### Thank you!
-
-More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
-Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet)
+- More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
+- Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet)
diff --git a/src/Config.py b/src/Config.py
index 6b2d5026..ec7a470d 100644
--- a/src/Config.py
+++ b/src/Config.py
@@ -3,7 +3,7 @@ import ConfigParser
class Config(object):
def __init__(self):
- self.version = "0.1.2"
+ self.version = "0.1.3"
self.parser = self.createArguments()
argv = sys.argv[:] # Copy command line arguments
argv = self.parseConfig(argv) # Add arguments from config file
diff --git a/src/Crypt/CryptHash.py b/src/Crypt/CryptHash.py
index 57932acc..d9de55f9 100644
--- a/src/Crypt/CryptHash.py
+++ b/src/Crypt/CryptHash.py
@@ -15,7 +15,7 @@ def sha512sum(file, blocksize=65536):
hash = hashlib.sha512()
for block in iter(lambda: file.read(blocksize), ""):
hash.update(block)
- return hash.hexdigest()
+ return hash.hexdigest()[0:64] # Truncate to 256bits is good enough
if __name__ == "__main__":
diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py
index 6a4bb6f9..7641e571 100644
--- a/src/Peer/Peer.py
+++ b/src/Peer/Peer.py
@@ -13,23 +13,32 @@ class Peer:
self.port = port
self.site = site
self.key = "%s:%s" % (ip, port)
+ self.log = None
self.socket = None
- self.last_found = None
+ self.last_found = None # Time of last found in the torrent tracker
+ self.last_response = None # Time of last successfull response from peer
+ self.last_ping = None # Last response time for ping
self.added = time.time()
- self.connection_error = 0
- self.hash_failed = 0
- self.download_bytes = 0
- self.download_time = 0
+ self.connection_error = 0 # Series of connection error
+ self.hash_failed = 0 # Number of bad files from peer
+ self.download_bytes = 0 # Bytes downloaded
+ self.download_time = 0 # Time spent to download
# Connect to host
def connect(self):
- self.log = logging.getLogger("Peer:%s:%s" % (self.ip, self.port))
+ if not self.log: self.log = logging.getLogger("Peer:%s:%s" % (self.ip, self.port))
+ if self.socket: self.socket.close()
+
self.socket = context.socket(zmq.REQ)
self.socket.setsockopt(zmq.SNDTIMEO, 5000) # Wait for data send
self.socket.setsockopt(zmq.LINGER, 500) # Wait for socket close
+ #self.socket.setsockopt(zmq.TCP_KEEPALIVE, 1) # Enable keepalive
+ #self.socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 4*60) # Send after 4 minute idle
+ #self.socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 15) # Wait 15 sec to response
+ #self.socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, 4) # 4 Probes
self.socket.connect('tcp://%s:%s' % (self.ip, self.port))
@@ -41,7 +50,10 @@ class Peer:
# Send a command to peer
def sendCmd(self, cmd, params = {}):
if not self.socket: self.connect()
- for retry in range(1,5):
+ if cmd != "ping" and self.last_response and time.time() - self.last_response > 20*60: # If last response if older than 20 minute, ping first to see if still alive
+ if not self.ping(): return None
+
+ for retry in range(1,3): # Retry 3 times
if config.debug_socket: self.log.debug("sendCmd: %s" % cmd)
try:
self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True))
@@ -53,11 +65,11 @@ class Peer:
self.onConnectionError()
else: # Successful request, reset connection error num
self.connection_error = 0
+ self.last_response = time.time()
return response
except Exception, err:
self.onConnectionError()
self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry))
- self.socket.close()
time.sleep(1*retry)
self.connect()
if type(err).__name__ == "Notify" and err.message == "Worker stopped": # Greenlet kill by worker
@@ -89,7 +101,24 @@ class Peer:
# Send a ping request
def ping(self):
- return self.sendCmd("ping")
+ response_time = None
+ for retry in range(1,3): # Retry 3 times
+ s = time.time()
+ with gevent.Timeout(10.0, False): # 10 sec timeout, dont raise exception
+ response = self.sendCmd("ping")
+ if response and "body" in response and response["body"] == "Pong!":
+ response_time = time.time()-s
+ break # All fine, exit from for loop
+ # Timeout reached or bad response
+ self.onConnectionError()
+ time.sleep(1)
+
+ if response_time:
+ self.log.debug("Ping: %.3f" % response_time)
+ else:
+ self.log.debug("Ping failed")
+ self.last_ping = response_time
+ return response_time
# Stop and remove from site
diff --git a/src/Site/Site.py b/src/Site/Site.py
index 6443b9db..44fbe3a7 100644
--- a/src/Site/Site.py
+++ b/src/Site/Site.py
@@ -27,7 +27,7 @@ class Site:
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.worker_manager = WorkerManager(self) # Handle site download from other peers
- self.bad_files = {} # SHA1 check failed files, need to redownload
+ self.bad_files = {} # SHA512 check failed files, need to redownload
self.content_updated = None # Content.js update time
self.last_downloads = [] # Files downloaded in run of self.download()
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
@@ -36,10 +36,16 @@ class Site:
self.loadContent(init=True) # Load content.json
self.loadSettings() # Load settings from sites.json
- if not self.settings.get("auth_key"):
- self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(12)) # To auth websocket
+ if not self.settings.get("auth_key"): # To auth user in site
+ self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
+
+ if not self.settings.get("wrapper_key"): # To auth websocket permissions
+ self.settings["wrapper_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
+ self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
+ self.saveSettings()
+
self.websockets = [] # Active site websocket connections
# Add event listeners
@@ -224,7 +230,7 @@ class Site:
for protocol, ip, port in SiteManager.TRACKERS:
if protocol == "udp":
- self.log.debug("Announing to %s://%s:%s..." % (protocol, ip, port))
+ self.log.debug("Announcing to %s://%s:%s..." % (protocol, ip, port))
tracker = UdpTrackerClient(ip, port)
tracker.peer_port = config.fileserver_port
try:
@@ -346,13 +352,17 @@ class Site:
else: # Check using sha1 hash
if self.content and inner_path in self.content["files"]:
- return CryptHash.sha1sum(file) == self.content["files"][inner_path]["sha1"]
+ if "sha512" in self.content["files"][inner_path]: # Use sha512 to verify if possible
+ return CryptHash.sha512sum(file) == self.content["files"][inner_path]["sha512"]
+ else: # Backward compatiblity
+ return CryptHash.sha1sum(file) == self.content["files"][inner_path]["sha1"]
+
else: # File not in content.json
self.log.error("File not in content.json: %s" % inner_path)
return False
- # Verify all files sha1sum using content.json
+ # Verify all files sha512sum using content.json
def verifyFiles(self, quick_check=False): # Fast = using file size
bad_files = []
if not self.content: # No content.json, download it first
@@ -396,17 +406,18 @@ class Site:
if file_name == "content.json" or (self.content["ignore"] and re.match(self.content["ignore"], file_path.replace(self.directory+"/", "") )): # Dont add content.json and ignore regexp pattern definied in content.json
self.log.info("- [SKIPPED] %s" % file_path)
else:
- sha1sum = CryptHash.sha1sum(file_path) # Calculate sha sum of file
+ sha1sum = CryptHash.sha1sum(file_path) # Calculate sha1 sum of file
+ sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
inner_path = re.sub("^%s/" % re.escape(self.directory), "", file_path)
- self.log.info("- %s (SHA1: %s)" % (file_path, sha1sum))
- hashed_files[inner_path] = {"sha1": sha1sum, "size": os.path.getsize(file_path)}
+ self.log.info("- %s (SHA512: %s)" % (file_path, sha512sum))
+ hashed_files[inner_path] = {"sha1": sha1sum, "sha512": sha512sum, "size": os.path.getsize(file_path)}
# Generate new content.json
- self.log.info("Adding timestamp and sha1sums to new content.json...")
+ self.log.info("Adding timestamp and sha512sums to new content.json...")
content = self.content.copy() # Create a copy of current content.json
- content["address"] = self.address # Add files sha1 hash
- content["files"] = hashed_files # Add files sha1 hash
+ content["address"] = self.address
+ content["files"] = hashed_files # Add files sha512 hash
content["modified"] = time.time() # Add timestamp
content["zeronet_version"] = config.version # Signer's zeronet version
del(content["sign"]) # Delete old sign
diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py
index 611173c6..4c57c916 100644
--- a/src/Ui/UiRequest.py
+++ b/src/Ui/UiRequest.py
@@ -122,7 +122,7 @@ class UiRequest:
inner_path=inner_path,
address=match.group("site"),
title=title,
- auth_key=site.settings["auth_key"],
+ wrapper_key=site.settings["wrapper_key"],
permissions=json.dumps(site.settings["permissions"]),
show_loadingscreen=json.dumps(not os.path.isfile(site.getPath(inner_path))),
homepage=config.homepage
@@ -209,13 +209,13 @@ class UiRequest:
def actionWebsocket(self):
ws = self.env.get("wsgi.websocket")
if ws:
- auth_key = self.get["auth_key"]
- # Find site by auth_key
+ wrapper_key = self.get["wrapper_key"]
+ # Find site by wraper_key
site = None
for site_check in self.server.sites.values():
- if site_check.settings["auth_key"] == auth_key: site = site_check
+ if site_check.settings["wrapper_key"] == wrapper_key: site = site_check
- if site: # Correct auth key
+ if site: # Correct wrapper key
ui_websocket = UiWebsocket(ws, site, self.server)
site.websockets.append(ui_websocket) # Add to site websockets to allow notify on events
ui_websocket.start()
@@ -223,8 +223,8 @@ class UiRequest:
if ui_websocket in site_check.websockets:
site_check.websockets.remove(ui_websocket)
return "Bye."
- else: # No site found by auth key
- self.log.error("Auth key not found: %s" % auth_key)
+ else: # No site found by wrapper key
+ self.log.error("Wrapper key not found: %s" % wraper_key)
return self.error403()
else:
start_response("400 Bad Request", [])
diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py
index 68d3ce77..634cd638 100644
--- a/src/Ui/UiWebsocket.py
+++ b/src/Ui/UiWebsocket.py
@@ -46,7 +46,7 @@ class UiWebsocket:
if channel in self.channels: # We are joined to channel
if channel == "siteChanged":
site = params[0] # Triggerer site
- site_info = self.siteInfo(site)
+ site_info = self.formatSiteInfo(site)
if len(params) > 1 and params[1]: # Extra data
site_info.update(params[1])
self.cmd("setSiteInfo", site_info)
@@ -121,18 +121,24 @@ class UiWebsocket:
# Format site info
- def siteInfo(self, site):
+ def formatSiteInfo(self, site):
+ content = site.content
+ if content and "files" in content: # Remove unnecessary data transfer
+ content = site.content.copy()
+ content["files"] = len(content["files"])
+ del(content["sign"])
+
ret = {
- "auth_id": self.site.settings["auth_key"][0:10],
- "auth_id_md5": hashlib.md5(self.site.settings["auth_key"][0:10]).hexdigest(),
+ "auth_key": self.site.settings["auth_key"],
+ "auth_key_sha512": hashlib.sha512(self.site.settings["auth_key"]).hexdigest()[0:64],
"address": site.address,
"settings": site.settings,
"content_updated": site.content_updated,
- "bad_files": site.bad_files.keys(),
- "last_downloads": site.last_downloads,
+ "bad_files": len(site.bad_files),
+ "last_downloads": len(site.last_downloads),
"peers": len(site.peers),
- "tasks": [task["inner_path"] for task in site.worker_manager.tasks],
- "content": site.content
+ "tasks": len([task["inner_path"] for task in site.worker_manager.tasks]),
+ "content": content
}
if site.settings["serving"] and site.content: ret["peers"] += 1 # Add myself if serving
return ret
@@ -140,7 +146,7 @@ class UiWebsocket:
# Send site details
def actionSiteInfo(self, to, params):
- ret = self.siteInfo(self.site)
+ ret = self.formatSiteInfo(self.site)
self.response(to, ret)
@@ -173,7 +179,7 @@ class UiWebsocket:
SiteManager.load() # Reload sites
for site in self.server.sites.values():
if not site.content: continue # Broken site
- ret.append(self.siteInfo(site))
+ ret.append(self.formatSiteInfo(site))
self.response(to, ret)
diff --git a/src/Ui/media/Wrapper.coffee b/src/Ui/media/Wrapper.coffee
index 6dfdffc2..b99a326e 100644
--- a/src/Ui/media/Wrapper.coffee
+++ b/src/Ui/media/Wrapper.coffee
@@ -118,8 +118,8 @@ class Wrapper
setSiteInfo: (site_info) ->
if site_info.event? # If loading screen visible add event to it
# File started downloading
- if site_info.event[0] == "file_added" and site_info.bad_files.length
- @loading.printLine("#{site_info.bad_files.length} files needs to be downloaded")
+ if site_info.event[0] == "file_added" and site_info.bad_files
+ @loading.printLine("#{site_info.bad_files} files needs to be downloaded")
# File finished downloading
else if site_info.event[0] == "file_done"
@loading.printLine("#{site_info.event[1]} downloaded")
@@ -148,5 +148,5 @@ class Wrapper
console.log "[Wrapper]", args...
-ws_url = "ws://#{window.location.hostname}:#{window.location.port}/Websocket?auth_key=#{window.auth_key}"
+ws_url = "ws://#{window.location.hostname}:#{window.location.port}/Websocket?wrapper_key=#{window.wrapper_key}"
window.wrapper = new Wrapper(ws_url)
diff --git a/src/Ui/media/all.js b/src/Ui/media/all.js
index 16923085..8c18be78 100644
--- a/src/Ui/media/all.js
+++ b/src/Ui/media/all.js
@@ -852,8 +852,8 @@ jQuery.extend( jQuery.easing,
Wrapper.prototype.setSiteInfo = function(site_info) {
if (site_info.event != null) {
- if (site_info.event[0] === "file_added" && site_info.bad_files.length) {
- this.loading.printLine("" + site_info.bad_files.length + " files needs to be downloaded");
+ if (site_info.event[0] === "file_added" && site_info.bad_files) {
+ this.loading.printLine("" + site_info.bad_files + " files needs to be downloaded");
} else if (site_info.event[0] === "file_done") {
this.loading.printLine("" + site_info.event[1] + " downloaded");
if (site_info.event[1] === window.inner_path) {
@@ -890,7 +890,7 @@ jQuery.extend( jQuery.easing,
})();
- ws_url = "ws://" + window.location.hostname + ":" + window.location.port + "/Websocket?auth_key=" + window.auth_key;
+ ws_url = "ws://" + window.location.hostname + ":" + window.location.port + "/Websocket?wrapper_key=" + window.wrapper_key;
window.wrapper = new Wrapper(ws_url);
diff --git a/src/Ui/template/wrapper.html b/src/Ui/template/wrapper.html
index 46c0cc0d..7de9dac0 100644
--- a/src/Ui/template/wrapper.html
+++ b/src/Ui/template/wrapper.html
@@ -35,12 +35,12 @@
-
+
-
+
diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py
index bbe9486c..b648f0da 100644
--- a/src/Worker/Worker.py
+++ b/src/Worker/Worker.py
@@ -50,12 +50,12 @@ class Worker:
self.manager.doneTask(task)
self.task = None
else: # Hash failed
+ self.manager.log.debug("%s: Hash failed: %s" % (self.key, task["inner_path"]))
self.task = None
self.peer.hash_failed += 1
if self.peer.hash_failed >= 3: # Broken peer
break
task["workers_num"] -= 1
- self.manager.log.error("%s: Hash failed: %s" % (self.key, task["inner_path"]))
time.sleep(1)
self.peer.onWorkerDone()
self.running = False
diff --git a/src/main.py b/src/main.py
index 791e952c..cc324e08 100644
--- a/src/main.py
+++ b/src/main.py
@@ -118,7 +118,7 @@ def siteVerify(address):
logging.info("Verifying site files...")
bad_files = site.verifyFiles()
if not bad_files:
- logging.info("[OK] All file sha1sum matches!")
+ logging.info("[OK] All file sha512sum matches!")
else:
logging.error("[ERROR] Error during verifying site files!")