2015-01-12 02:03:45 +01:00
|
|
|
import gevent, time, logging, shutil, os
|
|
|
|
from Peer import Peer
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
class Worker:
|
|
|
|
def __init__(self, manager, peer):
|
|
|
|
self.manager = manager
|
|
|
|
self.peer = peer
|
|
|
|
self.task = None
|
|
|
|
self.key = None
|
|
|
|
self.running = False
|
|
|
|
self.thread = None
|
|
|
|
|
|
|
|
|
2015-03-06 02:31:51 +01:00
|
|
|
def __str__(self):
|
|
|
|
return "Worker %s %s" % (self.manager.site.address_short, self.key)
|
|
|
|
|
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return "<%s>" % self.__str__()
|
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# Downloader thread
|
|
|
|
def downloader(self):
|
2015-01-13 21:19:40 +01:00
|
|
|
self.peer.hash_failed = 0 # Reset hash error counter
|
2015-01-12 02:03:45 +01:00
|
|
|
while self.running:
|
|
|
|
# Try to pickup free file download task
|
|
|
|
task = self.manager.getTask(self.peer)
|
|
|
|
if not task: # Die, no more task
|
|
|
|
self.manager.log.debug("%s: No task found, stopping" % self.key)
|
|
|
|
break
|
2015-01-16 11:52:42 +01:00
|
|
|
if not task["time_started"]: task["time_started"] = time.time() # Task started now
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
if task["workers_num"] > 0: # Wait a bit if someone already working on it
|
|
|
|
self.manager.log.debug("%s: Someone already working on %s, sleeping 1 sec..." % (self.key, task["inner_path"]))
|
|
|
|
time.sleep(1)
|
2015-01-14 22:57:43 +01:00
|
|
|
self.manager.log.debug("%s: %s, task done after sleep: %s" % (self.key, task["inner_path"], task["done"]))
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
if task["done"] == False:
|
|
|
|
self.task = task
|
2015-03-19 21:19:14 +01:00
|
|
|
site = task["site"]
|
2015-01-12 02:03:45 +01:00
|
|
|
task["workers_num"] += 1
|
2015-05-31 15:52:21 +02:00
|
|
|
try:
|
|
|
|
buff = self.peer.getFile(site.address, task["inner_path"])
|
|
|
|
except Exception, err:
|
|
|
|
self.manager.log.debug("%s: getFile error: err" % (self.key, err))
|
|
|
|
buff = None
|
2015-01-20 02:47:00 +01:00
|
|
|
if self.running == False: # Worker no longer needed or got killed
|
|
|
|
self.manager.log.debug("%s: No longer needed, returning: %s" % (self.key, task["inner_path"]))
|
2015-03-06 02:31:51 +01:00
|
|
|
break
|
2015-01-12 02:03:45 +01:00
|
|
|
if buff: # Download ok
|
2015-03-19 21:19:14 +01:00
|
|
|
correct = site.content_manager.verifyFile(task["inner_path"], buff)
|
2015-01-12 02:03:45 +01:00
|
|
|
else: # Download error
|
|
|
|
correct = False
|
|
|
|
if correct == True or correct == None: # Hash ok or same file
|
|
|
|
self.manager.log.debug("%s: Hash correct: %s" % (self.key, task["inner_path"]))
|
2015-03-19 21:19:14 +01:00
|
|
|
if correct == True and task["done"] == False: # Save if changed and task not done yet
|
2015-01-12 02:03:45 +01:00
|
|
|
buff.seek(0)
|
2015-03-19 21:19:14 +01:00
|
|
|
file_path = site.storage.getPath(task["inner_path"])
|
|
|
|
site.storage.write(task["inner_path"], buff)
|
|
|
|
if task["done"] == False: self.manager.doneTask(task)
|
|
|
|
task["workers_num"] -= 1
|
2015-01-12 02:03:45 +01:00
|
|
|
self.task = None
|
|
|
|
else: # Hash failed
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.manager.log.debug("%s: Hash failed: %s, failed peers: %s" % (self.key, task["inner_path"], len(task["failed"])))
|
2015-04-24 02:36:00 +02:00
|
|
|
task["failed"].append(self.peer)
|
2015-01-12 02:03:45 +01:00
|
|
|
self.task = None
|
|
|
|
self.peer.hash_failed += 1
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
if self.peer.hash_failed >= max(len(self.manager.tasks), 3): # More fails than tasks number but atleast 3: Broken peer
|
2015-01-12 02:03:45 +01:00
|
|
|
break
|
|
|
|
task["workers_num"] -= 1
|
|
|
|
time.sleep(1)
|
2015-01-13 21:19:40 +01:00
|
|
|
self.peer.onWorkerDone()
|
2015-01-12 02:03:45 +01:00
|
|
|
self.running = False
|
|
|
|
self.manager.removeWorker(self)
|
|
|
|
|
|
|
|
|
|
|
|
# Start the worker
|
|
|
|
def start(self):
|
|
|
|
self.running = True
|
|
|
|
self.thread = gevent.spawn(self.downloader)
|
|
|
|
|
2015-01-17 18:50:56 +01:00
|
|
|
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
# Skip current task
|
|
|
|
def skip(self):
|
|
|
|
self.manager.log.debug("%s: Force skipping" % self.key)
|
|
|
|
if self.thread:
|
|
|
|
self.thread.kill(exception=Debug.Notify("Worker stopped"))
|
|
|
|
self.start()
|
|
|
|
|
|
|
|
|
2015-01-17 18:50:56 +01:00
|
|
|
# Force stop the worker
|
2015-01-12 02:03:45 +01:00
|
|
|
def stop(self):
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
self.manager.log.debug("%s: Force stopping" % self.key)
|
2015-01-12 02:03:45 +01:00
|
|
|
self.running = False
|
2015-01-17 18:50:56 +01:00
|
|
|
if self.thread:
|
|
|
|
self.thread.kill(exception=Debug.Notify("Worker stopped"))
|
2015-03-06 02:31:51 +01:00
|
|
|
del self.thread
|
2015-01-12 02:03:45 +01:00
|
|
|
self.manager.removeWorker(self)
|