# Included modules import os import time import json import collections import itertools # Third party modules import gevent from Debug import Debug from Config import config from util import RateLimit from util import Msgpack from util import helper from Plugin import PluginManager from contextlib import closing FILE_BUFF = 1024 * 512 class RequestError(Exception): pass # Incoming requests @PluginManager.acceptPlugins class FileRequest(object): __slots__ = ("server", "connection", "req_id", "sites", "log", "responded") def __init__(self, server, connection): self.server = server self.connection = connection self.req_id = None self.sites = self.server.sites self.log = server.log self.responded = False # Responded to the request def send(self, msg, streaming=False): if not self.connection.closed: self.connection.send(msg, streaming) def sendRawfile(self, file, read_bytes): if not self.connection.closed: self.connection.sendRawfile(file, read_bytes) def response(self, msg, streaming=False): if self.responded: if config.verbose: self.log.debug("Req id %s already responded" % self.req_id) return if not isinstance(msg, dict): # If msg not a dict create a {"body": msg} msg = {"body": msg} msg["cmd"] = "response" msg["to"] = self.req_id self.responded = True self.send(msg, streaming=streaming) # Route file requests def route(self, cmd, req_id, params): self.req_id = req_id # Don't allow other sites than locked if "site" in params and self.connection.target_onion: valid_sites = self.connection.getValidSites() if params["site"] not in valid_sites and valid_sites != ["global"]: self.response({"error": "Invalid site"}) self.connection.log( "Site lock violation: %s not in %s, target onion: %s" % (params["site"], valid_sites, self.connection.target_onion) ) self.connection.badAction(5) return False if cmd == "update": event = "%s update %s %s" % (self.connection.id, params["site"], params["inner_path"]) # If called more than once within 15 sec only keep the last update RateLimit.callAsync(event, max(self.connection.bad_actions, 15), self.actionUpdate, params) else: func_name = "action" + cmd[0].upper() + cmd[1:] func = getattr(self, func_name, None) if cmd not in ["getFile", "streamFile"]: # Skip IO bound functions if self.connection.cpu_time > 0.5: self.log.debug( "Delay %s %s, cpu_time used by connection: %.3fs" % (self.connection.ip, cmd, self.connection.cpu_time) ) time.sleep(self.connection.cpu_time) if self.connection.cpu_time > 5: self.connection.close("Cpu time: %.3fs" % self.connection.cpu_time) s = time.time() if func: func(params) else: self.actionUnknown(cmd, params) if cmd not in ["getFile", "streamFile"]: taken = time.time() - s taken_sent = self.connection.last_sent_time - self.connection.last_send_time self.connection.cpu_time += taken - taken_sent # Update a site file request def actionUpdate(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(1) self.connection.badAction(5) return False inner_path = params.get("inner_path", "") current_content_modified = site.content_manager.contents.get(inner_path, {}).get("modified", 0) body = params["body"] if not inner_path.endswith("content.json"): self.response({"error": "Only content.json update allowed"}) self.connection.badAction(5) return should_validate_content = True if "modified" in params and params["modified"] <= current_content_modified: should_validate_content = False valid = None # Same or earlier content as we have elif not body: # No body sent, we have to download it first site.log.debug("Missing body from update for file %s, downloading ..." % inner_path) peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer try: body = peer.getFile(site.address, inner_path).read() except Exception as err: site.log.debug("Can't download updated file %s: %s" % (inner_path, err)) self.response({"error": "File invalid update: Can't download updaed file"}) self.connection.badAction(5) return if should_validate_content: try: content = json.loads(body.decode()) except Exception as err: site.log.debug("Update for %s is invalid JSON: %s" % (inner_path, err)) self.response({"error": "File invalid JSON"}) self.connection.badAction(5) return file_uri = "%s/%s:%s" % (site.address, inner_path, content["modified"]) if self.server.files_parsing.get(file_uri): # Check if we already working on it valid = None # Same file else: try: valid = site.content_manager.verifyFile(inner_path, content) except Exception as err: site.log.debug("Update for %s is invalid: %s" % (inner_path, err)) error = err valid = False if valid is True: # Valid and changed site.log.info("Update for %s looks valid, saving..." % inner_path) self.server.files_parsing[file_uri] = True site.storage.write(inner_path, body) del params["body"] site.onFileDone(inner_path) # Trigger filedone if inner_path.endswith("content.json"): # Download every changed file from peer peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer # On complete publish to other peers diffs = params.get("diffs", {}) site.onComplete.once(lambda: site.publish(inner_path=inner_path, diffs=diffs, limit=3), "publish_%s" % inner_path) # Load new content file and download changed files in new thread def downloader(): site.downloadContent(inner_path, peer=peer, diffs=params.get("diffs", {})) del self.server.files_parsing[file_uri] gevent.spawn(downloader) else: del self.server.files_parsing[file_uri] self.response({"ok": "Thanks, file %s updated!" % inner_path}) self.connection.goodAction() elif valid is None: # Not changed peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update old") # Add or get peer if peer: if not peer.connection: peer.connect(self.connection) # Assign current connection to peer if inner_path in site.content_manager.contents: peer.last_content_json_update = site.content_manager.contents[inner_path]["modified"] if config.verbose: site.log.debug( "Same version, adding new peer for locked files: %s, tasks: %s" % (peer.key, len(site.worker_manager.tasks)) ) for task in site.worker_manager.tasks: # New peer add to every ongoing task if task["peers"] and not task["optional_hash_id"]: # Download file from this peer too if its peer locked site.needFile(task["inner_path"], peer=peer, update=True, blocking=False) self.response({"ok": "File not changed"}) self.connection.badAction() else: # Invalid sign or sha hash self.response({"error": "File %s invalid: %s" % (inner_path, error)}) self.connection.badAction(5) def isReadable(self, site, inner_path, file, pos): return True # Send file content request def handleGetFile(self, params, streaming=False): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False try: file_path = site.storage.getPath(params["inner_path"]) if streaming: file_obj = site.storage.open(params["inner_path"]) else: file_obj = Msgpack.FilePart(file_path, "rb") with file_obj as file: file.seek(params["location"]) read_bytes = params.get("read_bytes", FILE_BUFF) file_size = os.fstat(file.fileno()).st_size if file_size > read_bytes: # Check if file is readable at current position (for big files) if not self.isReadable(site, params["inner_path"], file, params["location"]): raise RequestError("File not readable at position: %s" % params["location"]) else: if params.get("file_size") and params["file_size"] != file_size: self.connection.badAction(2) raise RequestError("File size does not match: %sB != %sB" % (params["file_size"], file_size)) if not streaming: file.read_bytes = read_bytes if params["location"] > file_size: self.connection.badAction(5) raise RequestError("Bad file location") if streaming: back = { "size": file_size, "location": min(file.tell() + read_bytes, file_size), "stream_bytes": min(read_bytes, file_size - params["location"]) } self.response(back) self.sendRawfile(file, read_bytes=read_bytes) else: back = { "body": file, "size": file_size, "location": min(file.tell() + file.read_bytes, file_size) } self.response(back, streaming=True) bytes_sent = min(read_bytes, file_size - params["location"]) # Number of bytes we going to send site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent if config.debug_socket: self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent)) # Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") if connected_peer: # Just added connected_peer.connect(self.connection) # Assign current connection to peer return {"bytes_sent": bytes_sent, "file_size": file_size, "location": params["location"]} except RequestError as err: self.log.debug("GetFile %s %s %s request error: %s" % (self.connection, params["site"], params["inner_path"], Debug.formatException(err))) self.response({"error": "File read error: %s" % err}) except OSError as err: if config.verbose: self.log.debug("GetFile read error: %s" % Debug.formatException(err)) self.response({"error": "File read error"}) return False except Exception as err: self.log.error("GetFile exception: %s" % Debug.formatException(err)) self.response({"error": "File read exception"}) return False def actionGetFile(self, params): return self.handleGetFile(params) def actionStreamFile(self, params): return self.handleGetFile(params, streaming=True) # Peer exchange request def actionPex(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False got_peer_keys = [] added = 0 # Add requester peer to site connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") if connected_peer: # It was not registered before added += 1 connected_peer.connect(self.connection) # Assign current connection to peer # Add sent peers to site for packed_address in itertools.chain(params.get("peers", []), params.get("peers_ipv6", [])): address = helper.unpackAddress(packed_address) got_peer_keys.append("%s:%s" % address) if site.addPeer(*address, source="pex"): added += 1 # Add sent onion peers to site for packed_address in params.get("peers_onion", []): address = helper.unpackOnionAddress(packed_address) got_peer_keys.append("%s:%s" % address) if site.addPeer(*address, source="pex"): added += 1 # Send back peers that is not in the sent list and connectable (not port 0) packed_peers = helper.packPeers(site.getConnectablePeers(params["need"], ignore=got_peer_keys, allow_private=False)) if added: site.worker_manager.onPeers() if config.verbose: self.log.debug( "Added %s peers to %s using pex, sending back %s" % (added, site, {key: len(val) for key, val in packed_peers.items()}) ) back = { "peers": packed_peers["ipv4"], "peers_ipv6": packed_peers["ipv6"], "peers_onion": packed_peers["onion"] } self.response(back) # Get modified content.json files since def actionListModified(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False modified_files = site.content_manager.listModified(params["since"]) # Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") if connected_peer: # Just added connected_peer.connect(self.connection) # Assign current connection to peer self.response({"modified_files": modified_files}) def actionGetHashfield(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False # Add peer to site if not added before peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="request") if not peer.connection: # Just added peer.connect(self.connection) # Assign current connection to peer peer.time_my_hashfield_sent = time.time() # Don't send again if not changed self.response({"hashfield_raw": site.content_manager.hashfield.tobytes()}) def findHashIds(self, site, hash_ids, limit=100): back = collections.defaultdict(lambda: collections.defaultdict(list)) found = site.worker_manager.findOptionalHashIds(hash_ids, limit=limit) for hash_id, peers in found.items(): for peer in peers: ip_type = helper.getIpType(peer.ip) if len(back[ip_type][hash_id]) < 20: back[ip_type][hash_id].append(peer.packMyAddress()) return back def actionFindHashIds(self, params): site = self.sites.get(params["site"]) s = time.time() if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False event_key = "%s_findHashIds_%s_%s" % (self.connection.ip, params["site"], len(params["hash_ids"])) if self.connection.cpu_time > 0.5 or not RateLimit.isAllowed(event_key, 60 * 5): time.sleep(0.1) back = self.findHashIds(site, params["hash_ids"], limit=10) else: back = self.findHashIds(site, params["hash_ids"]) RateLimit.called(event_key) my_hashes = [] my_hashfield_set = set(site.content_manager.hashfield) for hash_id in params["hash_ids"]: if hash_id in my_hashfield_set: my_hashes.append(hash_id) if config.verbose: self.log.debug( "Found: %s for %s hashids in %.3fs" % ({key: len(val) for key, val in back.items()}, len(params["hash_ids"]), time.time() - s) ) self.response({"peers": back["ipv4"], "peers_onion": back["onion"], "peers_ipv6": back["ipv6"], "my": my_hashes}) def actionSetHashfield(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False # Add or get peer peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, connection=self.connection, source="request") if not peer.connection: peer.connect(self.connection) peer.hashfield.replaceFromBytes(params["hashfield_raw"]) self.response({"ok": "Updated"}) # Send a simple Pong! answer def actionPing(self, params): self.response(b"Pong!") # Check requested port of the other peer def actionCheckport(self, params): if helper.getIpType(self.connection.ip) == "ipv6": sock_address = (self.connection.ip, params["port"], 0, 0) else: sock_address = (self.connection.ip, params["port"]) with closing(helper.createSocket(self.connection.ip)) as sock: sock.settimeout(5) if sock.connect_ex(sock_address) == 0: self.response({"status": "open", "ip_external": self.connection.ip}) else: self.response({"status": "closed", "ip_external": self.connection.ip}) # Unknown command def actionUnknown(self, cmd, params): self.response({"error": "Unknown command: %s" % cmd}) self.connection.badAction(5)