no need to specify content.json priority, its always downloaded first, peer broken after 3 hash failes, extra priority for css and js files

This commit is contained in:
HelloZeroNet 2015-01-14 22:57:43 +01:00
parent 776b572c82
commit bcd0c0025a
4 changed files with 22 additions and 17 deletions

View File

@ -42,6 +42,7 @@ Linux (Debian):
## How can I create a ZeroNet site? ## How can I create a ZeroNet site?
Shut down zeronet.py if you are running it already
``` ```
$ zeronet.py siteCreate $ zeronet.py siteCreate
... ...

View File

@ -189,7 +189,7 @@ class Site:
self.log.debug("Need content.json first") self.log.debug("Need content.json first")
self.announce() self.announce()
if inner_path != "content.json": # Prevent double download if inner_path != "content.json": # Prevent double download
task = self.worker_manager.addTask("content.json", peer, priority=99999) task = self.worker_manager.addTask("content.json", peer)
task.get() task.get()
self.loadContent() self.loadContent()
if not self.content: return False if not self.content: return False

View File

@ -24,8 +24,10 @@ class Worker:
if task["workers_num"] > 0: # Wait a bit if someone already working on it if task["workers_num"] > 0: # Wait a bit if someone already working on it
self.manager.log.debug("%s: Someone already working on %s, sleeping 1 sec..." % (self.key, task["inner_path"])) self.manager.log.debug("%s: Someone already working on %s, sleeping 1 sec..." % (self.key, task["inner_path"]))
time.sleep(1) time.sleep(1)
self.manager.log.debug("%s: %s, task done after sleep: %s" % (self.key, task["inner_path"], task["done"]))
if task["done"] == False: if task["done"] == False:
if not task["time_started"]: task["time_started"] = time.time() # Task started now
self.task = task self.task = task
task["workers_num"] += 1 task["workers_num"] += 1
buff = self.peer.getFile(task["site"].address, task["inner_path"]) buff = self.peer.getFile(task["site"].address, task["inner_path"])
@ -49,7 +51,7 @@ class Worker:
else: # Hash failed else: # Hash failed
self.task = None self.task = None
self.peer.hash_failed += 1 self.peer.hash_failed += 1
if self.peer.hash_failed > 5: # Broken peer if self.peer.hash_failed >= 3: # Broken peer
break break
task["workers_num"] -= 1 task["workers_num"] -= 1
self.manager.log.error("%s: Hash failed: %s" % (self.key, task["inner_path"])) self.manager.log.error("%s: Hash failed: %s" % (self.key, task["inner_path"]))

View File

@ -8,7 +8,7 @@ class WorkerManager:
def __init__(self, site): def __init__(self, site):
self.site = site self.site = site
self.workers = {} # Key: ip:port, Value: Worker.Worker self.workers = {} # Key: ip:port, Value: Worker.Worker
self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_start": time.time(), "peers": peers, "priority": 0} self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0}
self.log = logging.getLogger("WorkerManager:%s" % self.site.address_short) self.log = logging.getLogger("WorkerManager:%s" % self.site.address_short)
self.process_taskchecker = gevent.spawn(self.checkTasks) self.process_taskchecker = gevent.spawn(self.checkTasks)
@ -20,31 +20,33 @@ class WorkerManager:
if not self.tasks: continue if not self.tasks: continue
tasks = self.tasks[:] # Copy it so removing elements wont cause any problem tasks = self.tasks[:] # Copy it so removing elements wont cause any problem
for task in tasks: for task in tasks:
if time.time() >= task["time_start"]+60: # Task timed out if (task["time_started"] and time.time() >= task["time_started"]+60) or (time.time() >= task["time_added"]+60 and not self.workers): # Task taking too long time, kill it
self.log.debug("Cleaning up task: %s" % task) self.log.debug("Timeout, Cleaning up task: %s" % task)
# Clean up workers # Clean up workers
workers = self.findWorkers(task) workers = self.findWorkers(task)
for worker in workers: for worker in workers:
worker.stop() worker.stop()
# Remove task # Remove task
self.failTask(task) self.failTask(task)
elif time.time() >= task["time_start"]+15: # Task taking long time
self.log.debug("Task taking long time, find more peers: %s" % task["inner_path"]) elif (task["time_started"] and time.time() >= task["time_started"]+15) or not self.workers: # Task started more than 15 sec ago or no workers
task["site"].announce() # Find more peers self.log.debug("Task taking more than 15 secs, find more peers: %s" % task["inner_path"])
if task["peers"]: # Release the peer olck task["site"].announce() # Find more peers
self.log.debug("Task peer lock release: %s" % task["inner_path"]) if task["peers"]: # Release the peer olck
task["peers"] = [] self.log.debug("Task peer lock release: %s" % task["inner_path"])
self.startWorkers() task["peers"] = []
continue # One reannounce per loop self.startWorkers()
break # One reannounce per loop
# Tasks sorted by this # Tasks sorted by this
def taskSorter(self, task): def taskSorter(self, task):
if task["inner_path"] == "content.json": return 9999 # Content.json always prority if task["inner_path"] == "content.json": return 9999 # Content.json always prority
if task["inner_path"] == "index.html": return 9998 # index.html also important if task["inner_path"] == "index.html": return 9998 # index.html also important
return task["priority"]-task["workers_num"] # Prefer more priority and less workers priority = task["priority"]
if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"): priority += 1 # download js and css files first
return priority-task["workers_num"] # Prefer more priority and less workers
# Returns the next free or less worked task # Returns the next free or less worked task
@ -105,7 +107,7 @@ class WorkerManager:
peers = [peer] # Only download from this peer peers = [peer] # Only download from this peer
else: else:
peers = None peers = None
task = {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_start": time.time(), "peers": peers, "priority": priority} task = {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_added": time.time(), "time_started": None, "peers": peers, "priority": priority}
self.tasks.append(task) self.tasks.append(task)
self.log.debug("New task: %s" % task) self.log.debug("New task: %s" % task)
self.startWorkers() self.startWorkers()