2015-07-12 20:36:46 +02:00
|
|
|
import time
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
import gevent
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
from Debug import Debug
|
2016-04-06 13:39:17 +02:00
|
|
|
from Config import config
|
2015-03-06 02:31:51 +01:00
|
|
|
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
class Worker(object):
|
2015-03-06 02:31:51 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
def __init__(self, manager, peer):
|
|
|
|
self.manager = manager
|
|
|
|
self.peer = peer
|
|
|
|
self.task = None
|
|
|
|
self.key = None
|
|
|
|
self.running = False
|
|
|
|
self.thread = None
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
def __str__(self):
|
|
|
|
return "Worker %s %s" % (self.manager.site.address_short, self.key)
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
def __repr__(self):
|
|
|
|
return "<%s>" % self.__str__()
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Downloader thread
|
|
|
|
def downloader(self):
|
|
|
|
self.peer.hash_failed = 0 # Reset hash error counter
|
|
|
|
while self.running:
|
|
|
|
# Try to pickup free file download task
|
|
|
|
task = self.manager.getTask(self.peer)
|
|
|
|
if not task: # Die, no more task
|
|
|
|
self.manager.log.debug("%s: No task found, stopping" % self.key)
|
|
|
|
break
|
|
|
|
if not task["time_started"]:
|
|
|
|
task["time_started"] = time.time() # Task started now
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if task["workers_num"] > 0: # Wait a bit if someone already working on it
|
2016-04-06 13:39:17 +02:00
|
|
|
if config.verbose:
|
2017-10-03 16:02:49 +02:00
|
|
|
self.manager.log.debug("%s: Someone already working on %s (pri: %s), sleeping 1 sec..." % (self.key, task["inner_path"], task["priority"]))
|
|
|
|
for sleep_i in range(1,10):
|
|
|
|
time.sleep(0.1)
|
|
|
|
if task["done"] or task["workers_num"] == 0:
|
|
|
|
if config.verbose:
|
|
|
|
self.manager.log.debug("%s: %s, picked task free after %ss sleep. (done: %s)" % (self.key, task["inner_path"], 0.1 * sleep_i, task["done"]))
|
|
|
|
break
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if task["done"] is False:
|
|
|
|
self.task = task
|
|
|
|
site = task["site"]
|
|
|
|
task["workers_num"] += 1
|
|
|
|
try:
|
2017-06-19 16:10:18 +02:00
|
|
|
buff = self.peer.getFile(site.address, task["inner_path"], task["size"])
|
2015-07-12 20:36:46 +02:00
|
|
|
except Exception, err:
|
2015-07-25 13:38:58 +02:00
|
|
|
self.manager.log.debug("%s: getFile error: %s" % (self.key, err))
|
2015-07-12 20:36:46 +02:00
|
|
|
buff = None
|
2015-07-31 23:34:53 +02:00
|
|
|
if self.running is False: # Worker no longer needed or got killed
|
2015-07-12 20:36:46 +02:00
|
|
|
self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"]))
|
|
|
|
break
|
2015-07-31 23:34:53 +02:00
|
|
|
if task["done"] is True: # Task done, try to find new one
|
|
|
|
continue
|
2015-07-12 20:36:46 +02:00
|
|
|
if buff: # Download ok
|
2017-06-19 16:10:49 +02:00
|
|
|
try:
|
|
|
|
correct = site.content_manager.verifyFile(task["inner_path"], buff)
|
|
|
|
except Exception, err:
|
|
|
|
correct = False
|
2015-07-12 20:36:46 +02:00
|
|
|
else: # Download error
|
2017-06-19 16:10:49 +02:00
|
|
|
err = "Download failed"
|
2015-07-12 20:36:46 +02:00
|
|
|
correct = False
|
2017-06-19 16:10:49 +02:00
|
|
|
if correct is True or correct is None: # Verify ok or same file
|
|
|
|
self.manager.log.debug("%s: Verify correct: %s" % (self.key, task["inner_path"]))
|
2015-07-12 20:36:46 +02:00
|
|
|
if correct is True and task["done"] is False: # Save if changed and task not done yet
|
|
|
|
buff.seek(0)
|
|
|
|
site.storage.write(task["inner_path"], buff)
|
|
|
|
if task["done"] is False:
|
|
|
|
self.manager.doneTask(task)
|
|
|
|
task["workers_num"] -= 1
|
|
|
|
self.task = None
|
2017-06-19 16:10:49 +02:00
|
|
|
else: # Verify failed
|
2015-07-12 20:36:46 +02:00
|
|
|
self.manager.log.debug(
|
2017-06-19 16:10:49 +02:00
|
|
|
"%s: Verify failed: %s, error: %s, failed peers: %s" %
|
|
|
|
(self.key, task["inner_path"], err, len(task["failed"]))
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
|
|
|
task["failed"].append(self.peer)
|
|
|
|
self.task = None
|
|
|
|
self.peer.hash_failed += 1
|
2015-11-15 11:13:57 +01:00
|
|
|
if self.peer.hash_failed >= max(len(self.manager.tasks), 3) or self.peer.connection_error > 10:
|
2015-07-12 20:36:46 +02:00
|
|
|
# Broken peer: More fails than tasks number but atleast 3
|
|
|
|
break
|
|
|
|
task["workers_num"] -= 1
|
|
|
|
time.sleep(1)
|
|
|
|
self.peer.onWorkerDone()
|
|
|
|
self.running = False
|
|
|
|
self.manager.removeWorker(self)
|
2015-01-17 18:50:56 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Start the worker
|
|
|
|
def start(self):
|
|
|
|
self.running = True
|
|
|
|
self.thread = gevent.spawn(self.downloader)
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Skip current task
|
|
|
|
def skip(self):
|
|
|
|
self.manager.log.debug("%s: Force skipping" % self.key)
|
|
|
|
if self.thread:
|
|
|
|
self.thread.kill(exception=Debug.Notify("Worker stopped"))
|
|
|
|
self.start()
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Force stop the worker
|
|
|
|
def stop(self):
|
|
|
|
self.manager.log.debug("%s: Force stopping" % self.key)
|
|
|
|
self.running = False
|
|
|
|
if self.thread:
|
|
|
|
self.thread.kill(exception=Debug.Notify("Worker stopped"))
|
|
|
|
del self.thread
|
|
|
|
self.manager.removeWorker(self)
|