2015-04-13 23:08:57 +02:00
|
|
|
import os, msgpack, shutil, gevent, socket, struct, random
|
2015-01-12 02:03:45 +01:00
|
|
|
from cStringIO import StringIO
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
2015-01-21 12:58:26 +01:00
|
|
|
from Config import config
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
FILE_BUFF = 1024*512
|
|
|
|
|
|
|
|
# Request from me
|
|
|
|
class FileRequest:
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
def __init__(self, server, connection):
|
|
|
|
self.server = server
|
|
|
|
self.connection = connection
|
|
|
|
|
|
|
|
self.req_id = None
|
|
|
|
self.sites = self.server.sites
|
|
|
|
self.log = server.log
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
2015-04-13 23:08:57 +02:00
|
|
|
def unpackAddress(self, packed):
|
|
|
|
return (socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0])
|
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
def send(self, msg):
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.connection.send(msg)
|
|
|
|
|
|
|
|
|
|
|
|
def response(self, msg):
|
2015-01-12 02:03:45 +01:00
|
|
|
if not isinstance(msg, dict): # If msg not a dict create a {"body": msg}
|
|
|
|
msg = {"body": msg}
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
msg["cmd"] = "response"
|
|
|
|
msg["to"] = self.req_id
|
|
|
|
self.send(msg)
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Route file requests
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
def route(self, cmd, req_id, params):
|
|
|
|
self.req_id = req_id
|
2015-01-12 02:03:45 +01:00
|
|
|
if cmd == "getFile":
|
|
|
|
self.actionGetFile(params)
|
|
|
|
elif cmd == "update":
|
|
|
|
self.actionUpdate(params)
|
2015-04-13 23:08:57 +02:00
|
|
|
elif cmd == "pex":
|
|
|
|
self.actionPex(params)
|
2015-01-12 02:03:45 +01:00
|
|
|
elif cmd == "ping":
|
|
|
|
self.actionPing()
|
|
|
|
else:
|
|
|
|
self.actionUnknown(cmd, params)
|
|
|
|
|
|
|
|
|
|
|
|
# Update a site file request
|
|
|
|
def actionUpdate(self, params):
|
|
|
|
site = self.sites.get(params["site"])
|
|
|
|
if not site or not site.settings["serving"]: # Site unknown or not serving
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "Unknown site"})
|
2015-01-12 02:03:45 +01:00
|
|
|
return False
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if site.settings["own"] and params["inner_path"].endswith("content.json"):
|
|
|
|
self.log.debug("Someone trying to push a file to own site %s, reload local %s first" % (site.address, params["inner_path"]))
|
2015-02-14 14:05:00 +01:00
|
|
|
changed = site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
|
|
|
|
if changed: # Content.json changed locally
|
|
|
|
site.settings["size"] = site.content_manager.getTotalSize() # Update site size
|
2015-01-12 02:03:45 +01:00
|
|
|
buff = StringIO(params["body"])
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
valid = site.content_manager.verifyFile(params["inner_path"], buff)
|
2015-01-12 02:03:45 +01:00
|
|
|
if valid == True: # Valid and changed
|
2015-03-15 00:48:06 +01:00
|
|
|
self.log.info("Update for %s looks valid, saving..." % params["inner_path"])
|
2015-01-12 02:03:45 +01:00
|
|
|
buff.seek(0)
|
2015-03-19 21:19:14 +01:00
|
|
|
site.storage.write(params["inner_path"], buff)
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
site.onFileDone(params["inner_path"]) # Trigger filedone
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if params["inner_path"].endswith("content.json"): # Download every changed file from peer
|
2015-04-14 02:37:31 +02:00
|
|
|
peer = site.addPeer(self.connection.ip, self.connection.port, return_peer = True) # Add or get peer
|
2015-04-08 01:57:55 +02:00
|
|
|
site.onComplete.once(lambda: site.publish(inner_path=params["inner_path"]), "publish_%s" % params["inner_path"]) # On complete publish to other peers
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
gevent.spawn(
|
|
|
|
lambda: site.downloadContent(params["inner_path"], peer=peer)
|
|
|
|
) # Load new content file and download changed files in new thread
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"ok": "Thanks, file %s updated!" % params["inner_path"]})
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
elif valid == None: # Not changed
|
|
|
|
peer = site.addPeer(*params["peer"], return_peer = True) # Add or get peer
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if peer:
|
|
|
|
self.log.debug("Same version, adding new peer for locked files: %s, tasks: %s" % (peer.key, len(site.worker_manager.tasks)) )
|
|
|
|
for task in site.worker_manager.tasks: # New peer add to every ongoing task
|
|
|
|
if task["peers"]: site.needFile(task["inner_path"], peer=peer, update=True, blocking=False) # Download file from this peer too if its peer locked
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"ok": "File not changed"})
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
else: # Invalid sign or sha1 hash
|
2015-01-15 23:24:51 +01:00
|
|
|
self.log.debug("Update for %s is invalid" % params["inner_path"])
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "File invalid"})
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Send file content request
|
|
|
|
def actionGetFile(self, params):
|
|
|
|
site = self.sites.get(params["site"])
|
|
|
|
if not site or not site.settings["serving"]: # Site unknown or not serving
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "Unknown site"})
|
2015-01-12 02:03:45 +01:00
|
|
|
return False
|
|
|
|
try:
|
2015-03-19 21:19:14 +01:00
|
|
|
file_path = site.storage.getPath(params["inner_path"])
|
2015-01-21 12:58:26 +01:00
|
|
|
if config.debug_socket: self.log.debug("Opening file: %s" % file_path)
|
|
|
|
file = open(file_path, "rb")
|
2015-01-12 02:03:45 +01:00
|
|
|
file.seek(params["location"])
|
|
|
|
back = {}
|
|
|
|
back["body"] = file.read(FILE_BUFF)
|
|
|
|
back["location"] = file.tell()
|
|
|
|
back["size"] = os.fstat(file.fileno()).st_size
|
2015-01-21 12:58:26 +01:00
|
|
|
if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"]))
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response(back)
|
2015-01-21 12:58:26 +01:00
|
|
|
if config.debug_socket: self.log.debug("File %s sent" % file_path)
|
2015-04-12 23:59:22 +02:00
|
|
|
|
|
|
|
# Add peer to site if not added before
|
2015-04-13 23:08:57 +02:00
|
|
|
site.addPeer(self.connection.ip, self.connection.port)
|
2015-01-12 02:03:45 +01:00
|
|
|
except Exception, err:
|
2015-02-26 01:32:27 +01:00
|
|
|
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "File read error: %s" % Debug.formatException(err)})
|
2015-01-12 02:03:45 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
|
2015-04-13 23:08:57 +02:00
|
|
|
# Peer exchange request
|
|
|
|
def actionPex(self, params):
|
|
|
|
site = self.sites.get(params["site"])
|
|
|
|
if not site or not site.settings["serving"]: # Site unknown or not serving
|
|
|
|
self.response({"error": "Unknown site"})
|
|
|
|
return False
|
|
|
|
|
|
|
|
got_peer_keys = []
|
|
|
|
added = 0
|
|
|
|
site.addPeer(self.connection.ip, self.connection.port) # Add requester peer to site
|
|
|
|
for peer in params["peers"]: # Add sent peers to site
|
|
|
|
address = self.unpackAddress(peer)
|
|
|
|
got_peer_keys.append("%s:%s" % address)
|
|
|
|
if (site.addPeer(*address)): added += 1
|
|
|
|
# Send back peers that is not in the sent list
|
|
|
|
peers = site.peers.values()
|
|
|
|
random.shuffle(peers)
|
|
|
|
packed_peers = [peer.packAddress() for peer in peers if peer.key not in got_peer_keys][0:params["need"]]
|
|
|
|
if added:
|
2015-04-14 02:37:31 +02:00
|
|
|
self.log.debug("Added %s peers to %s using pex, sending back %s" % (added, site, len(packed_peers)))
|
2015-04-13 23:08:57 +02:00
|
|
|
self.response({"peers": packed_peers})
|
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# Send a simple Pong! answer
|
|
|
|
def actionPing(self):
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response("Pong!")
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Unknown command
|
|
|
|
def actionUnknown(self, cmd, params):
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "Unknown command: %s" % cmd})
|