2015-04-13 23:08:57 +02:00
|
|
|
import os, msgpack, shutil, gevent, socket, struct, random
|
2015-01-12 02:03:45 +01:00
|
|
|
from cStringIO import StringIO
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
2015-01-21 12:58:26 +01:00
|
|
|
from Config import config
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
from util import RateLimit, StreamingMsgpack
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
FILE_BUFF = 1024*512
|
|
|
|
|
|
|
|
# Request from me
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
class FileRequest(object):
|
|
|
|
__slots__ = ("server", "connection", "req_id", "sites", "log", "responded")
|
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
def __init__(self, server, connection):
|
|
|
|
self.server = server
|
|
|
|
self.connection = connection
|
|
|
|
|
|
|
|
self.req_id = None
|
|
|
|
self.sites = self.server.sites
|
|
|
|
self.log = server.log
|
2015-04-24 02:36:00 +02:00
|
|
|
self.responded = False # Responded to the request
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
2015-04-13 23:08:57 +02:00
|
|
|
def unpackAddress(self, packed):
|
|
|
|
return (socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0])
|
|
|
|
|
|
|
|
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
def send(self, msg, streaming=False):
|
2015-04-24 02:36:00 +02:00
|
|
|
if not self.connection.closed:
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
self.connection.send(msg, streaming)
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
|
|
|
|
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
def response(self, msg, streaming=False):
|
2015-04-24 02:36:00 +02:00
|
|
|
if self.responded:
|
|
|
|
self.log.debug("Req id %s already responded" % self.req_id)
|
|
|
|
return
|
2015-01-12 02:03:45 +01:00
|
|
|
if not isinstance(msg, dict): # If msg not a dict create a {"body": msg}
|
|
|
|
msg = {"body": msg}
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
msg["cmd"] = "response"
|
|
|
|
msg["to"] = self.req_id
|
2015-04-24 02:36:00 +02:00
|
|
|
self.responded = True
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
self.send(msg, streaming=streaming)
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Route file requests
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
def route(self, cmd, req_id, params):
|
|
|
|
self.req_id = req_id
|
2015-04-24 02:36:00 +02:00
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
if cmd == "getFile":
|
|
|
|
self.actionGetFile(params)
|
|
|
|
elif cmd == "update":
|
2015-04-24 02:36:00 +02:00
|
|
|
event = "%s update %s %s" % (self.connection.id, params["site"], params["inner_path"])
|
|
|
|
if not RateLimit.isAllowed(event): # There was already an updat for this file in the last 10 second
|
|
|
|
self.response({"ok": "File update queued"})
|
|
|
|
RateLimit.callAsync(event, 10, self.actionUpdate, params) # If called more than once within 10 sec only keep the last update
|
|
|
|
|
2015-04-13 23:08:57 +02:00
|
|
|
elif cmd == "pex":
|
|
|
|
self.actionPex(params)
|
2015-04-30 11:49:03 +02:00
|
|
|
elif cmd == "listModified":
|
|
|
|
self.actionListModified(params)
|
2015-01-12 02:03:45 +01:00
|
|
|
elif cmd == "ping":
|
|
|
|
self.actionPing()
|
|
|
|
else:
|
|
|
|
self.actionUnknown(cmd, params)
|
|
|
|
|
|
|
|
|
|
|
|
# Update a site file request
|
|
|
|
def actionUpdate(self, params):
|
|
|
|
site = self.sites.get(params["site"])
|
|
|
|
if not site or not site.settings["serving"]: # Site unknown or not serving
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "Unknown site"})
|
2015-01-12 02:03:45 +01:00
|
|
|
return False
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if site.settings["own"] and params["inner_path"].endswith("content.json"):
|
|
|
|
self.log.debug("Someone trying to push a file to own site %s, reload local %s first" % (site.address, params["inner_path"]))
|
2015-02-14 14:05:00 +01:00
|
|
|
changed = site.content_manager.loadContent(params["inner_path"], add_bad_files=False)
|
|
|
|
if changed: # Content.json changed locally
|
|
|
|
site.settings["size"] = site.content_manager.getTotalSize() # Update site size
|
2015-01-12 02:03:45 +01:00
|
|
|
buff = StringIO(params["body"])
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
valid = site.content_manager.verifyFile(params["inner_path"], buff)
|
2015-01-12 02:03:45 +01:00
|
|
|
if valid == True: # Valid and changed
|
2015-03-15 00:48:06 +01:00
|
|
|
self.log.info("Update for %s looks valid, saving..." % params["inner_path"])
|
2015-01-12 02:03:45 +01:00
|
|
|
buff.seek(0)
|
2015-03-19 21:19:14 +01:00
|
|
|
site.storage.write(params["inner_path"], buff)
|
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
site.onFileDone(params["inner_path"]) # Trigger filedone
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if params["inner_path"].endswith("content.json"): # Download every changed file from peer
|
2015-04-14 02:37:31 +02:00
|
|
|
peer = site.addPeer(self.connection.ip, self.connection.port, return_peer = True) # Add or get peer
|
2015-04-08 01:57:55 +02:00
|
|
|
site.onComplete.once(lambda: site.publish(inner_path=params["inner_path"]), "publish_%s" % params["inner_path"]) # On complete publish to other peers
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
gevent.spawn(
|
|
|
|
lambda: site.downloadContent(params["inner_path"], peer=peer)
|
|
|
|
) # Load new content file and download changed files in new thread
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"ok": "Thanks, file %s updated!" % params["inner_path"]})
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
elif valid == None: # Not changed
|
|
|
|
peer = site.addPeer(*params["peer"], return_peer = True) # Add or get peer
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
if peer:
|
|
|
|
self.log.debug("Same version, adding new peer for locked files: %s, tasks: %s" % (peer.key, len(site.worker_manager.tasks)) )
|
|
|
|
for task in site.worker_manager.tasks: # New peer add to every ongoing task
|
|
|
|
if task["peers"]: site.needFile(task["inner_path"], peer=peer, update=True, blocking=False) # Download file from this peer too if its peer locked
|
2015-01-12 02:03:45 +01:00
|
|
|
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"ok": "File not changed"})
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
else: # Invalid sign or sha1 hash
|
2015-01-15 23:24:51 +01:00
|
|
|
self.log.debug("Update for %s is invalid" % params["inner_path"])
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "File invalid"})
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Send file content request
|
|
|
|
def actionGetFile(self, params):
|
|
|
|
site = self.sites.get(params["site"])
|
|
|
|
if not site or not site.settings["serving"]: # Site unknown or not serving
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "Unknown site"})
|
2015-01-12 02:03:45 +01:00
|
|
|
return False
|
|
|
|
try:
|
2015-03-19 21:19:14 +01:00
|
|
|
file_path = site.storage.getPath(params["inner_path"])
|
2015-01-21 12:58:26 +01:00
|
|
|
if config.debug_socket: self.log.debug("Opening file: %s" % file_path)
|
rev134, Removed ZeroMQ dependency and support, GC after every stat page, GC call stat command, Streaming files directly to socket without msgpack overhead, Use listModified to query changed content.json files, Fix urllib memory leak onolder pythons, Fix security tests, Sitemanager testsuite, Announce on site resume, Site publish serves files max 60s
2015-05-03 13:06:43 +02:00
|
|
|
with StreamingMsgpack.FilePart(file_path, "rb") as file:
|
|
|
|
file.seek(params["location"])
|
|
|
|
file.read_bytes = FILE_BUFF
|
|
|
|
back = {}
|
|
|
|
back["body"] = file
|
|
|
|
back["size"] = os.fstat(file.fileno()).st_size
|
|
|
|
back["location"] = min(file.tell()+FILE_BUFF, back["size"])
|
|
|
|
if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"]))
|
|
|
|
self.response(back, streaming=True)
|
2015-01-21 12:58:26 +01:00
|
|
|
if config.debug_socket: self.log.debug("File %s sent" % file_path)
|
2015-04-12 23:59:22 +02:00
|
|
|
|
|
|
|
# Add peer to site if not added before
|
2015-04-17 23:12:22 +02:00
|
|
|
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
|
|
|
|
if connected_peer: # Just added
|
|
|
|
connected_peer.connect(self.connection) # Assign current connection to peer
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
except Exception, err:
|
2015-02-26 01:32:27 +01:00
|
|
|
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "File read error: %s" % Debug.formatException(err)})
|
2015-01-12 02:03:45 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
|
2015-04-13 23:08:57 +02:00
|
|
|
# Peer exchange request
|
|
|
|
def actionPex(self, params):
|
|
|
|
site = self.sites.get(params["site"])
|
|
|
|
if not site or not site.settings["serving"]: # Site unknown or not serving
|
|
|
|
self.response({"error": "Unknown site"})
|
|
|
|
return False
|
|
|
|
|
|
|
|
got_peer_keys = []
|
|
|
|
added = 0
|
2015-04-17 23:12:22 +02:00
|
|
|
connected_peer = site.addPeer(self.connection.ip, self.connection.port) # Add requester peer to site
|
|
|
|
if connected_peer: # Just added
|
|
|
|
added +=1
|
|
|
|
connected_peer.connect(self.connection) # Assign current connection to peer
|
|
|
|
|
2015-04-13 23:08:57 +02:00
|
|
|
for peer in params["peers"]: # Add sent peers to site
|
|
|
|
address = self.unpackAddress(peer)
|
|
|
|
got_peer_keys.append("%s:%s" % address)
|
2015-05-31 15:52:21 +02:00
|
|
|
if site.addPeer(*address): added += 1
|
2015-04-15 02:54:10 +02:00
|
|
|
# Send back peers that is not in the sent list and connectable (not port 0)
|
2015-04-15 23:33:21 +02:00
|
|
|
packed_peers = [peer.packAddress() for peer in site.getConnectablePeers(params["need"], got_peer_keys)]
|
2015-04-13 23:08:57 +02:00
|
|
|
if added:
|
2015-04-17 23:12:22 +02:00
|
|
|
site.worker_manager.onPeers()
|
2015-04-14 02:37:31 +02:00
|
|
|
self.log.debug("Added %s peers to %s using pex, sending back %s" % (added, site, len(packed_peers)))
|
2015-04-13 23:08:57 +02:00
|
|
|
self.response({"peers": packed_peers})
|
|
|
|
|
|
|
|
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
# Get modified content.json files since
|
2015-04-30 11:49:03 +02:00
|
|
|
def actionListModified(self, params):
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
site = self.sites.get(params["site"])
|
|
|
|
if not site or not site.settings["serving"]: # Site unknown or not serving
|
|
|
|
self.response({"error": "Unknown site"})
|
|
|
|
return False
|
|
|
|
modified_files = {inner_path: content["modified"] for inner_path, content in site.content_manager.contents.iteritems() if content["modified"] > params["since"]}
|
version 0.3.0, rev187, Trusted authorization sites support, --publish option on signing, cryptSign command line option, OpenSSL enabled on OSX, Crypto verify allows list of valid addresses, Option for version 2 json DB tables, DbCursor SELECT parameters bugfix, Add peer to site on ListModified, Download blind includes when new site added, Publish command better messages, Multi-threaded announce, New http Torrent trackers, Wait for dbschema.json on query, Handle json import errors, More compact writeJson storage command, Testcase for signing and verifying, Workaround to make non target=_top links work, More clean UiWebsocket command route, Send cert_user_id on siteinfo, Notify other local clients on local file modify, Option to wait for file download before sql query, File rules websocket API command, Cert add and select, set websocket API command, Put focus on innerframe, innerloaded wrapper api command to add hashtag, Allow more file error on big sites, Keep worker running after stuked on done task, New more stable openSSL layer that works on OSX, Noparallel parameter bugfix, RateLimit allowed again interval bugfix, Updater skips non-writeable files, Try to close openssl dll before update
2015-05-25 01:26:33 +02:00
|
|
|
|
|
|
|
# Add peer to site if not added before
|
|
|
|
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
|
|
|
|
if connected_peer: # Just added
|
|
|
|
connected_peer.connect(self.connection) # Assign current connection to peer
|
|
|
|
|
rev125, Class statistics, OpenSSL disabled on OSX by default because of possible segfault, --disable_openssl command line parameter, Save memory on Connection, Peer and FileRequest objects using slots, Dont store modification time from the far future, Able to query modified files from peer, Allow reannounce in 30secs, Use with command in SiteStorage, Always create dir before write file, PeerCmd shell command to query specific command from peer
2015-04-29 23:12:45 +02:00
|
|
|
self.response({"modified_files": modified_files})
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
# Send a simple Pong! answer
|
|
|
|
def actionPing(self):
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response("Pong!")
|
2015-01-12 02:03:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Unknown command
|
|
|
|
def actionUnknown(self, cmd, params):
|
version 0.2.4, peerPing and peerGetFile commands, old content update bugfix, new network code and protocol, connection share between sites, connection reuse, dont retry bad file more than 3 times in 20 min, multi threaded include file download, shuffle peers before publish, simple internal stats page, dont retry on failed peers, more than 10 peers publish bugfix
2015-02-23 23:33:31 +01:00
|
|
|
self.response({"error": "Unknown command: %s" % cmd})
|