diff --git a/README.md b/README.md index bceb0446..b96c85f2 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Linux (Debian): ## How can I create a ZeroNet site? +Shut down zeronet.py if you are running it already ``` $ zeronet.py siteCreate ... diff --git a/src/Site/Site.py b/src/Site/Site.py index 44c73df3..b25affca 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -189,7 +189,7 @@ class Site: self.log.debug("Need content.json first") self.announce() if inner_path != "content.json": # Prevent double download - task = self.worker_manager.addTask("content.json", peer, priority=99999) + task = self.worker_manager.addTask("content.json", peer) task.get() self.loadContent() if not self.content: return False diff --git a/src/Worker/Worker.py b/src/Worker/Worker.py index 018f98c7..239d2394 100644 --- a/src/Worker/Worker.py +++ b/src/Worker/Worker.py @@ -24,8 +24,10 @@ class Worker: if task["workers_num"] > 0: # Wait a bit if someone already working on it self.manager.log.debug("%s: Someone already working on %s, sleeping 1 sec..." % (self.key, task["inner_path"])) time.sleep(1) + self.manager.log.debug("%s: %s, task done after sleep: %s" % (self.key, task["inner_path"], task["done"])) if task["done"] == False: + if not task["time_started"]: task["time_started"] = time.time() # Task started now self.task = task task["workers_num"] += 1 buff = self.peer.getFile(task["site"].address, task["inner_path"]) @@ -49,7 +51,7 @@ class Worker: else: # Hash failed self.task = None self.peer.hash_failed += 1 - if self.peer.hash_failed > 5: # Broken peer + if self.peer.hash_failed >= 3: # Broken peer break task["workers_num"] -= 1 self.manager.log.error("%s: Hash failed: %s" % (self.key, task["inner_path"])) diff --git a/src/Worker/WorkerManager.py b/src/Worker/WorkerManager.py index d5e63fe8..4409fbf4 100644 --- a/src/Worker/WorkerManager.py +++ b/src/Worker/WorkerManager.py @@ -8,7 +8,7 @@ class WorkerManager: def __init__(self, site): self.site = site self.workers = {} # Key: ip:port, Value: Worker.Worker - self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_start": time.time(), "peers": peers, "priority": 0} + self.tasks = [] # {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_started": None, "time_added": time.time(), "peers": peers, "priority": 0} self.log = logging.getLogger("WorkerManager:%s" % self.site.address_short) self.process_taskchecker = gevent.spawn(self.checkTasks) @@ -20,31 +20,33 @@ class WorkerManager: if not self.tasks: continue tasks = self.tasks[:] # Copy it so removing elements wont cause any problem for task in tasks: - if time.time() >= task["time_start"]+60: # Task timed out - self.log.debug("Cleaning up task: %s" % task) - + if (task["time_started"] and time.time() >= task["time_started"]+60) or (time.time() >= task["time_added"]+60 and not self.workers): # Task taking too long time, kill it + self.log.debug("Timeout, Cleaning up task: %s" % task) # Clean up workers workers = self.findWorkers(task) for worker in workers: worker.stop() - # Remove task self.failTask(task) - elif time.time() >= task["time_start"]+15: # Task taking long time - self.log.debug("Task taking long time, find more peers: %s" % task["inner_path"]) - task["site"].announce() # Find more peers - if task["peers"]: # Release the peer olck - self.log.debug("Task peer lock release: %s" % task["inner_path"]) - task["peers"] = [] - self.startWorkers() - continue # One reannounce per loop + + elif (task["time_started"] and time.time() >= task["time_started"]+15) or not self.workers: # Task started more than 15 sec ago or no workers + self.log.debug("Task taking more than 15 secs, find more peers: %s" % task["inner_path"]) + task["site"].announce() # Find more peers + if task["peers"]: # Release the peer olck + self.log.debug("Task peer lock release: %s" % task["inner_path"]) + task["peers"] = [] + self.startWorkers() + break # One reannounce per loop + # Tasks sorted by this def taskSorter(self, task): if task["inner_path"] == "content.json": return 9999 # Content.json always prority if task["inner_path"] == "index.html": return 9998 # index.html also important - return task["priority"]-task["workers_num"] # Prefer more priority and less workers + priority = task["priority"] + if task["inner_path"].endswith(".js") or task["inner_path"].endswith(".css"): priority += 1 # download js and css files first + return priority-task["workers_num"] # Prefer more priority and less workers # Returns the next free or less worked task @@ -105,7 +107,7 @@ class WorkerManager: peers = [peer] # Only download from this peer else: peers = None - task = {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_start": time.time(), "peers": peers, "priority": priority} + task = {"evt": evt, "workers_num": 0, "site": self.site, "inner_path": inner_path, "done": False, "time_added": time.time(), "time_started": None, "peers": peers, "priority": priority} self.tasks.append(task) self.log.debug("New task: %s" % task) self.startWorkers()