version 0.1.3, tructate sha512 to 256bits, retry peer cmd only 3 times, ping peer before cmd to find stucked sockets, ping with timeout and retry, separate wrapper_key and auth_key, changed sha1 to sha512, backward compatibility to sha1, reduce websocket bw usage on events, removed wrapper hash from wrapper iframe url
This commit is contained in:
HelloZeroNet
parent
b37e309eda
commit
014a79912f
10
README.md
10
README.md
|
|
@ -16,7 +16,7 @@ Decentralized websites using Bitcoin crypto and BitTorrent network
|
|||
- When you visit a new zeronet site, it's trying to find peers using BitTorrent network and download the site files (html, css, js...) from them.
|
||||
- Each visited sites become also served by You.
|
||||
- Every site containing a `site.json` which holds all other files sha1 hash and a sign generated using site's private key.
|
||||
- If the site owner (who has the private key for the site address) modifies the site, then he/she signs the new `content.json` and publish it to the peers. After the peers have verified the `content.json` integrity using the sign they download the modified files and publish the new content to other peers.
|
||||
- If the site owner (who has the private key for the site address) modifies the site, then he/she signs the new `content.json` and publish it to the peers. After the peers have verified the `content.json` integrity (using the sign), they download the modified files and publish the new content to other peers.
|
||||
|
||||
|
||||
## Screenshot
|
||||
|
|
@ -62,6 +62,8 @@ $ zeronet.py
|
|||
```
|
||||
Congratulations, you are done! Now anyone can access your site using http://localhost:43110/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2
|
||||
|
||||
Next steps: [ZeroNet Developer Documentation](https://github.com/HelloZeroNet/ZeroNet/wiki/ZeroNet-Developer-Documentation)
|
||||
|
||||
|
||||
## How can I modify a ZeroNet site?
|
||||
- Modify files located in data/13DNDkMUExRf9Xa9ogwPKqp7zyHFEqbhC2 directory. After you done:
|
||||
|
|
@ -85,8 +87,8 @@ Site:13DNDk..bhC2 Successfuly published to 3 peers
|
|||
|
||||
Bitcoin: 1QDhxQ6PraUZa21ET5fYUCPgdrwBomnFgX
|
||||
|
||||
|
||||
#### Thank you!
|
||||
|
||||
|
||||
More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
|
||||
Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet)
|
||||
- More info, help, changelog, zeronet sites: http://www.reddit.com/r/zeronet/
|
||||
- Come, chat with us: [#zeronet @ FreeNode](https://kiwiirc.com/client/irc.freenode.net/zeronet)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import ConfigParser
|
|||
|
||||
class Config(object):
|
||||
def __init__(self):
|
||||
self.version = "0.1.2"
|
||||
self.version = "0.1.3"
|
||||
self.parser = self.createArguments()
|
||||
argv = sys.argv[:] # Copy command line arguments
|
||||
argv = self.parseConfig(argv) # Add arguments from config file
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ def sha512sum(file, blocksize=65536):
|
|||
hash = hashlib.sha512()
|
||||
for block in iter(lambda: file.read(blocksize), ""):
|
||||
hash.update(block)
|
||||
return hash.hexdigest()
|
||||
return hash.hexdigest()[0:64] # Truncate to 256bits is good enough
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -13,23 +13,32 @@ class Peer:
|
|||
self.port = port
|
||||
self.site = site
|
||||
self.key = "%s:%s" % (ip, port)
|
||||
self.log = None
|
||||
|
||||
self.socket = None
|
||||
self.last_found = None
|
||||
self.last_found = None # Time of last found in the torrent tracker
|
||||
self.last_response = None # Time of last successfull response from peer
|
||||
self.last_ping = None # Last response time for ping
|
||||
self.added = time.time()
|
||||
|
||||
self.connection_error = 0
|
||||
self.hash_failed = 0
|
||||
self.download_bytes = 0
|
||||
self.download_time = 0
|
||||
self.connection_error = 0 # Series of connection error
|
||||
self.hash_failed = 0 # Number of bad files from peer
|
||||
self.download_bytes = 0 # Bytes downloaded
|
||||
self.download_time = 0 # Time spent to download
|
||||
|
||||
|
||||
# Connect to host
|
||||
def connect(self):
|
||||
self.log = logging.getLogger("Peer:%s:%s" % (self.ip, self.port))
|
||||
if not self.log: self.log = logging.getLogger("Peer:%s:%s" % (self.ip, self.port))
|
||||
if self.socket: self.socket.close()
|
||||
|
||||
self.socket = context.socket(zmq.REQ)
|
||||
self.socket.setsockopt(zmq.SNDTIMEO, 5000) # Wait for data send
|
||||
self.socket.setsockopt(zmq.LINGER, 500) # Wait for socket close
|
||||
#self.socket.setsockopt(zmq.TCP_KEEPALIVE, 1) # Enable keepalive
|
||||
#self.socket.setsockopt(zmq.TCP_KEEPALIVE_IDLE, 4*60) # Send after 4 minute idle
|
||||
#self.socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, 15) # Wait 15 sec to response
|
||||
#self.socket.setsockopt(zmq.TCP_KEEPALIVE_CNT, 4) # 4 Probes
|
||||
self.socket.connect('tcp://%s:%s' % (self.ip, self.port))
|
||||
|
||||
|
||||
|
|
@ -41,7 +50,10 @@ class Peer:
|
|||
# Send a command to peer
|
||||
def sendCmd(self, cmd, params = {}):
|
||||
if not self.socket: self.connect()
|
||||
for retry in range(1,5):
|
||||
if cmd != "ping" and self.last_response and time.time() - self.last_response > 20*60: # If last response if older than 20 minute, ping first to see if still alive
|
||||
if not self.ping(): return None
|
||||
|
||||
for retry in range(1,3): # Retry 3 times
|
||||
if config.debug_socket: self.log.debug("sendCmd: %s" % cmd)
|
||||
try:
|
||||
self.socket.send(msgpack.packb({"cmd": cmd, "params": params}, use_bin_type=True))
|
||||
|
|
@ -53,11 +65,11 @@ class Peer:
|
|||
self.onConnectionError()
|
||||
else: # Successful request, reset connection error num
|
||||
self.connection_error = 0
|
||||
self.last_response = time.time()
|
||||
return response
|
||||
except Exception, err:
|
||||
self.onConnectionError()
|
||||
self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry))
|
||||
self.socket.close()
|
||||
time.sleep(1*retry)
|
||||
self.connect()
|
||||
if type(err).__name__ == "Notify" and err.message == "Worker stopped": # Greenlet kill by worker
|
||||
|
|
@ -89,7 +101,24 @@ class Peer:
|
|||
|
||||
# Send a ping request
|
||||
def ping(self):
|
||||
return self.sendCmd("ping")
|
||||
response_time = None
|
||||
for retry in range(1,3): # Retry 3 times
|
||||
s = time.time()
|
||||
with gevent.Timeout(10.0, False): # 10 sec timeout, dont raise exception
|
||||
response = self.sendCmd("ping")
|
||||
if response and "body" in response and response["body"] == "Pong!":
|
||||
response_time = time.time()-s
|
||||
break # All fine, exit from for loop
|
||||
# Timeout reached or bad response
|
||||
self.onConnectionError()
|
||||
time.sleep(1)
|
||||
|
||||
if response_time:
|
||||
self.log.debug("Ping: %.3f" % response_time)
|
||||
else:
|
||||
self.log.debug("Ping failed")
|
||||
self.last_ping = response_time
|
||||
return response_time
|
||||
|
||||
|
||||
# Stop and remove from site
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class Site:
|
|||
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
|
||||
self.last_announce = 0 # Last announce time to tracker
|
||||
self.worker_manager = WorkerManager(self) # Handle site download from other peers
|
||||
self.bad_files = {} # SHA1 check failed files, need to redownload
|
||||
self.bad_files = {} # SHA512 check failed files, need to redownload
|
||||
self.content_updated = None # Content.js update time
|
||||
self.last_downloads = [] # Files downloaded in run of self.download()
|
||||
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
|
||||
|
|
@ -36,10 +36,16 @@ class Site:
|
|||
self.loadContent(init=True) # Load content.json
|
||||
self.loadSettings() # Load settings from sites.json
|
||||
|
||||
if not self.settings.get("auth_key"):
|
||||
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(12)) # To auth websocket
|
||||
if not self.settings.get("auth_key"): # To auth user in site
|
||||
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
|
||||
self.log.debug("New auth key: %s" % self.settings["auth_key"])
|
||||
self.saveSettings()
|
||||
|
||||
if not self.settings.get("wrapper_key"): # To auth websocket permissions
|
||||
self.settings["wrapper_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
|
||||
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
|
||||
self.saveSettings()
|
||||
|
||||
self.websockets = [] # Active site websocket connections
|
||||
|
||||
# Add event listeners
|
||||
|
|
@ -224,7 +230,7 @@ class Site:
|
|||
|
||||
for protocol, ip, port in SiteManager.TRACKERS:
|
||||
if protocol == "udp":
|
||||
self.log.debug("Announing to %s://%s:%s..." % (protocol, ip, port))
|
||||
self.log.debug("Announcing to %s://%s:%s..." % (protocol, ip, port))
|
||||
tracker = UdpTrackerClient(ip, port)
|
||||
tracker.peer_port = config.fileserver_port
|
||||
try:
|
||||
|
|
@ -346,13 +352,17 @@ class Site:
|
|||
|
||||
else: # Check using sha1 hash
|
||||
if self.content and inner_path in self.content["files"]:
|
||||
return CryptHash.sha1sum(file) == self.content["files"][inner_path]["sha1"]
|
||||
if "sha512" in self.content["files"][inner_path]: # Use sha512 to verify if possible
|
||||
return CryptHash.sha512sum(file) == self.content["files"][inner_path]["sha512"]
|
||||
else: # Backward compatiblity
|
||||
return CryptHash.sha1sum(file) == self.content["files"][inner_path]["sha1"]
|
||||
|
||||
else: # File not in content.json
|
||||
self.log.error("File not in content.json: %s" % inner_path)
|
||||
return False
|
||||
|
||||
|
||||
# Verify all files sha1sum using content.json
|
||||
# Verify all files sha512sum using content.json
|
||||
def verifyFiles(self, quick_check=False): # Fast = using file size
|
||||
bad_files = []
|
||||
if not self.content: # No content.json, download it first
|
||||
|
|
@ -396,17 +406,18 @@ class Site:
|
|||
if file_name == "content.json" or (self.content["ignore"] and re.match(self.content["ignore"], file_path.replace(self.directory+"/", "") )): # Dont add content.json and ignore regexp pattern definied in content.json
|
||||
self.log.info("- [SKIPPED] %s" % file_path)
|
||||
else:
|
||||
sha1sum = CryptHash.sha1sum(file_path) # Calculate sha sum of file
|
||||
sha1sum = CryptHash.sha1sum(file_path) # Calculate sha1 sum of file
|
||||
sha512sum = CryptHash.sha512sum(file_path) # Calculate sha512 sum of file
|
||||
inner_path = re.sub("^%s/" % re.escape(self.directory), "", file_path)
|
||||
self.log.info("- %s (SHA1: %s)" % (file_path, sha1sum))
|
||||
hashed_files[inner_path] = {"sha1": sha1sum, "size": os.path.getsize(file_path)}
|
||||
self.log.info("- %s (SHA512: %s)" % (file_path, sha512sum))
|
||||
hashed_files[inner_path] = {"sha1": sha1sum, "sha512": sha512sum, "size": os.path.getsize(file_path)}
|
||||
|
||||
# Generate new content.json
|
||||
self.log.info("Adding timestamp and sha1sums to new content.json...")
|
||||
self.log.info("Adding timestamp and sha512sums to new content.json...")
|
||||
|
||||
content = self.content.copy() # Create a copy of current content.json
|
||||
content["address"] = self.address # Add files sha1 hash
|
||||
content["files"] = hashed_files # Add files sha1 hash
|
||||
content["address"] = self.address
|
||||
content["files"] = hashed_files # Add files sha512 hash
|
||||
content["modified"] = time.time() # Add timestamp
|
||||
content["zeronet_version"] = config.version # Signer's zeronet version
|
||||
del(content["sign"]) # Delete old sign
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ class UiRequest:
|
|||
inner_path=inner_path,
|
||||
address=match.group("site"),
|
||||
title=title,
|
||||
auth_key=site.settings["auth_key"],
|
||||
wrapper_key=site.settings["wrapper_key"],
|
||||
permissions=json.dumps(site.settings["permissions"]),
|
||||
show_loadingscreen=json.dumps(not os.path.isfile(site.getPath(inner_path))),
|
||||
homepage=config.homepage
|
||||
|
|
@ -209,13 +209,13 @@ class UiRequest:
|
|||
def actionWebsocket(self):
|
||||
ws = self.env.get("wsgi.websocket")
|
||||
if ws:
|
||||
auth_key = self.get["auth_key"]
|
||||
# Find site by auth_key
|
||||
wrapper_key = self.get["wrapper_key"]
|
||||
# Find site by wraper_key
|
||||
site = None
|
||||
for site_check in self.server.sites.values():
|
||||
if site_check.settings["auth_key"] == auth_key: site = site_check
|
||||
if site_check.settings["wrapper_key"] == wrapper_key: site = site_check
|
||||
|
||||
if site: # Correct auth key
|
||||
if site: # Correct wrapper key
|
||||
ui_websocket = UiWebsocket(ws, site, self.server)
|
||||
site.websockets.append(ui_websocket) # Add to site websockets to allow notify on events
|
||||
ui_websocket.start()
|
||||
|
|
@ -223,8 +223,8 @@ class UiRequest:
|
|||
if ui_websocket in site_check.websockets:
|
||||
site_check.websockets.remove(ui_websocket)
|
||||
return "Bye."
|
||||
else: # No site found by auth key
|
||||
self.log.error("Auth key not found: %s" % auth_key)
|
||||
else: # No site found by wrapper key
|
||||
self.log.error("Wrapper key not found: %s" % wraper_key)
|
||||
return self.error403()
|
||||
else:
|
||||
start_response("400 Bad Request", [])
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class UiWebsocket:
|
|||
if channel in self.channels: # We are joined to channel
|
||||
if channel == "siteChanged":
|
||||
site = params[0] # Triggerer site
|
||||
site_info = self.siteInfo(site)
|
||||
site_info = self.formatSiteInfo(site)
|
||||
if len(params) > 1 and params[1]: # Extra data
|
||||
site_info.update(params[1])
|
||||
self.cmd("setSiteInfo", site_info)
|
||||
|
|
@ -121,18 +121,24 @@ class UiWebsocket:
|
|||
|
||||
|
||||
# Format site info
|
||||
def siteInfo(self, site):
|
||||
def formatSiteInfo(self, site):
|
||||
content = site.content
|
||||
if content and "files" in content: # Remove unnecessary data transfer
|
||||
content = site.content.copy()
|
||||
content["files"] = len(content["files"])
|
||||
del(content["sign"])
|
||||
|
||||
ret = {
|
||||
"auth_id": self.site.settings["auth_key"][0:10],
|
||||
"auth_id_md5": hashlib.md5(self.site.settings["auth_key"][0:10]).hexdigest(),
|
||||
"auth_key": self.site.settings["auth_key"],
|
||||
"auth_key_sha512": hashlib.sha512(self.site.settings["auth_key"]).hexdigest()[0:64],
|
||||
"address": site.address,
|
||||
"settings": site.settings,
|
||||
"content_updated": site.content_updated,
|
||||
"bad_files": site.bad_files.keys(),
|
||||
"last_downloads": site.last_downloads,
|
||||
"bad_files": len(site.bad_files),
|
||||
"last_downloads": len(site.last_downloads),
|
||||
"peers": len(site.peers),
|
||||
"tasks": [task["inner_path"] for task in site.worker_manager.tasks],
|
||||
"content": site.content
|
||||
"tasks": len([task["inner_path"] for task in site.worker_manager.tasks]),
|
||||
"content": content
|
||||
}
|
||||
if site.settings["serving"] and site.content: ret["peers"] += 1 # Add myself if serving
|
||||
return ret
|
||||
|
|
@ -140,7 +146,7 @@ class UiWebsocket:
|
|||
|
||||
# Send site details
|
||||
def actionSiteInfo(self, to, params):
|
||||
ret = self.siteInfo(self.site)
|
||||
ret = self.formatSiteInfo(self.site)
|
||||
self.response(to, ret)
|
||||
|
||||
|
||||
|
|
@ -173,7 +179,7 @@ class UiWebsocket:
|
|||
SiteManager.load() # Reload sites
|
||||
for site in self.server.sites.values():
|
||||
if not site.content: continue # Broken site
|
||||
ret.append(self.siteInfo(site))
|
||||
ret.append(self.formatSiteInfo(site))
|
||||
self.response(to, ret)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -118,8 +118,8 @@ class Wrapper
|
|||
setSiteInfo: (site_info) ->
|
||||
if site_info.event? # If loading screen visible add event to it
|
||||
# File started downloading
|
||||
if site_info.event[0] == "file_added" and site_info.bad_files.length
|
||||
@loading.printLine("#{site_info.bad_files.length} files needs to be downloaded")
|
||||
if site_info.event[0] == "file_added" and site_info.bad_files
|
||||
@loading.printLine("#{site_info.bad_files} files needs to be downloaded")
|
||||
# File finished downloading
|
||||
else if site_info.event[0] == "file_done"
|
||||
@loading.printLine("#{site_info.event[1]} downloaded")
|
||||
|
|
@ -148,5 +148,5 @@ class Wrapper
|
|||
console.log "[Wrapper]", args...
|
||||
|
||||
|
||||
ws_url = "ws://#{window.location.hostname}:#{window.location.port}/Websocket?auth_key=#{window.auth_key}"
|
||||
ws_url = "ws://#{window.location.hostname}:#{window.location.port}/Websocket?wrapper_key=#{window.wrapper_key}"
|
||||
window.wrapper = new Wrapper(ws_url)
|
||||
|
|
|
|||
|
|
@ -852,8 +852,8 @@ jQuery.extend( jQuery.easing,
|
|||
|
||||
Wrapper.prototype.setSiteInfo = function(site_info) {
|
||||
if (site_info.event != null) {
|
||||
if (site_info.event[0] === "file_added" && site_info.bad_files.length) {
|
||||
this.loading.printLine("" + site_info.bad_files.length + " files needs to be downloaded");
|
||||
if (site_info.event[0] === "file_added" && site_info.bad_files) {
|
||||
this.loading.printLine("" + site_info.bad_files + " files needs to be downloaded");
|
||||
} else if (site_info.event[0] === "file_done") {
|
||||
this.loading.printLine("" + site_info.event[1] + " downloaded");
|
||||
if (site_info.event[1] === window.inner_path) {
|
||||
|
|
@ -890,7 +890,7 @@ jQuery.extend( jQuery.easing,
|
|||
|
||||
})();
|
||||
|
||||
ws_url = "ws://" + window.location.hostname + ":" + window.location.port + "/Websocket?auth_key=" + window.auth_key;
|
||||
ws_url = "ws://" + window.location.hostname + ":" + window.location.port + "/Websocket?wrapper_key=" + window.wrapper_key;
|
||||
|
||||
window.wrapper = new Wrapper(ws_url);
|
||||
|
||||
|
|
|
|||
|
|
@ -35,12 +35,12 @@
|
|||
|
||||
|
||||
<!-- Site Iframe -->
|
||||
<iframe src='/media/{address}/{inner_path}#auth_key={auth_key}' id='inner-iframe' sandbox="allow-forms allow-scripts allow-top-navigation"></iframe>
|
||||
<iframe src='/media/{address}/{inner_path}' id='inner-iframe' sandbox="allow-forms allow-scripts allow-top-navigation"></iframe>
|
||||
|
||||
|
||||
<!-- Site info -->
|
||||
<script>address = "{address}"</script>
|
||||
<script>auth_key = "{auth_key}"</script>
|
||||
<script>wrapper_key = "{wrapper_key}"</script>
|
||||
<script>inner_path = "{inner_path}"</script>
|
||||
<script>permissions = {permissions}</script>
|
||||
<script>show_loadingscreen = {show_loadingscreen}</script>
|
||||
|
|
|
|||
|
|
@ -50,12 +50,12 @@ class Worker:
|
|||
self.manager.doneTask(task)
|
||||
self.task = None
|
||||
else: # Hash failed
|
||||
self.manager.log.debug("%s: Hash failed: %s" % (self.key, task["inner_path"]))
|
||||
self.task = None
|
||||
self.peer.hash_failed += 1
|
||||
if self.peer.hash_failed >= 3: # Broken peer
|
||||
break
|
||||
task["workers_num"] -= 1
|
||||
self.manager.log.error("%s: Hash failed: %s" % (self.key, task["inner_path"]))
|
||||
time.sleep(1)
|
||||
self.peer.onWorkerDone()
|
||||
self.running = False
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ def siteVerify(address):
|
|||
logging.info("Verifying site files...")
|
||||
bad_files = site.verifyFiles()
|
||||
if not bad_files:
|
||||
logging.info("[OK] All file sha1sum matches!")
|
||||
logging.info("[OK] All file sha512sum matches!")
|
||||
else:
|
||||
logging.error("[ERROR] Error during verifying site files!")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue