2015-07-12 20:36:46 +02:00
|
|
|
import os
|
|
|
|
import json
|
|
|
|
import logging
|
|
|
|
import hashlib
|
|
|
|
import re
|
|
|
|
import time
|
|
|
|
import random
|
|
|
|
import sys
|
|
|
|
import struct
|
|
|
|
import socket
|
|
|
|
import urllib
|
|
|
|
import urllib2
|
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
import gevent
|
2016-09-04 17:57:32 +02:00
|
|
|
import gevent.pool
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-01-12 02:03:45 +01:00
|
|
|
import util
|
2015-07-12 20:36:46 +02:00
|
|
|
from lib import bencode
|
|
|
|
from lib.subtl.subtl import UdpTrackerClient
|
2015-01-12 02:03:45 +01:00
|
|
|
from Config import config
|
|
|
|
from Peer import Peer
|
|
|
|
from Worker import WorkerManager
|
2015-01-17 18:50:56 +01:00
|
|
|
from Debug import Debug
|
version 0.2.0, new lib for bitcoin ecc, dont display or track notify errors, dont reload again within 1 sec, null peer ip fix, signingmoved to ContentManager, content.json include support, content.json multisig ready, content.json proper bitcoincore compatible signing, content.json include permissions, multithreaded publish, publish timeout 60s, no exception on invalid bitcoin address, testcase for new lib, bip32 based persite privatekey generation, multiuser ready, simple json database query command, websocket api fileGet, wrapper loading title stuck bugfix
2015-02-09 02:09:02 +01:00
|
|
|
from Content import ContentManager
|
2015-03-19 21:19:14 +01:00
|
|
|
from SiteStorage import SiteStorage
|
2015-09-13 23:17:13 +02:00
|
|
|
from Crypt import CryptHash
|
2015-09-27 02:08:53 +02:00
|
|
|
from util import helper
|
2016-04-06 14:01:03 +02:00
|
|
|
from util import Diff
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
from Plugin import PluginManager
|
2015-01-12 02:03:45 +01:00
|
|
|
import SiteManager
|
|
|
|
|
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
@PluginManager.acceptPlugins
|
|
|
|
class Site(object):
|
2015-01-12 02:03:45 +01:00
|
|
|
|
2016-09-04 17:51:55 +02:00
|
|
|
def __init__(self, address, allow_create=True, settings=None):
|
2015-07-12 20:36:46 +02:00
|
|
|
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
|
|
|
|
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
|
|
|
|
self.log = logging.getLogger("Site:%s" % self.address_short)
|
2015-10-11 02:22:53 +02:00
|
|
|
self.addEventListeners()
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
self.content = None # Load content.json
|
|
|
|
self.peers = {} # Key: ip:port, Value: Peer.Peer
|
|
|
|
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_announce = 0 # Last announce time to tracker
|
2015-09-10 23:25:09 +02:00
|
|
|
self.last_tracker_id = random.randint(0, 10) # Last announced tracker id
|
2015-07-12 20:36:46 +02:00
|
|
|
self.worker_manager = WorkerManager(self) # Handle site download from other peers
|
|
|
|
self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
|
|
|
|
self.content_updated = None # Content.js update time
|
|
|
|
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
|
|
|
|
self.page_requested = False # Page viewed in browser
|
2016-09-05 13:55:51 +02:00
|
|
|
self.websockets = [] # Active site websocket connections
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-09-05 13:55:51 +02:00
|
|
|
self.connection_server = None
|
2016-09-04 17:51:55 +02:00
|
|
|
self.loadSettings(settings) # Load settings from sites.json
|
2017-10-04 12:50:24 +02:00
|
|
|
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
|
2016-09-04 17:41:04 +02:00
|
|
|
self.content_manager = ContentManager(self)
|
|
|
|
self.content_manager.loadContents() # Load content.json files
|
2015-09-24 22:08:08 +02:00
|
|
|
if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible
|
|
|
|
self.connection_server = sys.modules["main"].file_server
|
|
|
|
else:
|
|
|
|
self.connection_server = None
|
2015-07-12 20:36:46 +02:00
|
|
|
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
|
2015-09-13 23:17:13 +02:00
|
|
|
self.settings["auth_key"] = CryptHash.random()
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug("New auth key: %s" % self.settings["auth_key"])
|
|
|
|
|
|
|
|
if not self.settings.get("wrapper_key"): # To auth websocket permissions
|
2015-09-13 23:17:13 +02:00
|
|
|
self.settings["wrapper_key"] = CryptHash.random()
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
|
|
|
|
|
2017-08-15 02:40:38 +02:00
|
|
|
if not self.settings.get("ajax_key"): # To auth websocket permissions
|
|
|
|
self.settings["ajax_key"] = CryptHash.random()
|
|
|
|
self.log.debug("New ajax key: %s" % self.settings["ajax_key"])
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
def __str__(self):
|
|
|
|
return "Site %s" % self.address_short
|
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return "<%s>" % self.__str__()
|
|
|
|
|
|
|
|
# Load site settings from data/sites.json
|
2016-09-04 17:43:30 +02:00
|
|
|
def loadSettings(self, settings=None):
|
|
|
|
if not settings:
|
|
|
|
settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address)
|
|
|
|
if settings:
|
|
|
|
self.settings = settings
|
|
|
|
if "cache" not in settings:
|
|
|
|
settings["cache"] = {}
|
2016-11-07 23:36:04 +01:00
|
|
|
if "size_files_optional" not in settings:
|
|
|
|
settings["size_optional"] = 0
|
|
|
|
if "optional_downloaded" not in settings:
|
|
|
|
settings["optional_downloaded"] = 0
|
2016-09-04 17:43:30 +02:00
|
|
|
self.bad_files = settings["cache"].get("bad_files", {})
|
|
|
|
settings["cache"]["bad_files"] = {}
|
2017-10-11 14:29:54 +02:00
|
|
|
# Give it minimum 10 tries after restart
|
2016-09-04 17:43:30 +02:00
|
|
|
for inner_path in self.bad_files:
|
2017-10-11 14:29:54 +02:00
|
|
|
self.bad_files[inner_path] = min(self.bad_files[inner_path], 20)
|
2015-07-12 20:36:46 +02:00
|
|
|
else:
|
2017-05-23 12:40:42 +02:00
|
|
|
self.settings = {
|
|
|
|
"own": False, "serving": True, "permissions": [],
|
|
|
|
"added": int(time.time()), "optional_downloaded": 0, "size_optional": 0
|
|
|
|
} # Default
|
2017-07-31 14:39:21 +02:00
|
|
|
if config.download_optional == "auto":
|
|
|
|
self.settings["autodownloadoptional"] = True
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
|
|
|
|
# Add admin permissions to homepage
|
|
|
|
if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]:
|
|
|
|
self.settings["permissions"].append("ADMIN")
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
return
|
|
|
|
|
|
|
|
# Save site settings to data/sites.json
|
|
|
|
def saveSettings(self):
|
2016-09-06 14:52:55 +02:00
|
|
|
if not SiteManager.site_manager.sites:
|
|
|
|
SiteManager.site_manager.sites = {}
|
|
|
|
if not SiteManager.site_manager.sites.get(self.address):
|
2016-08-29 11:17:19 +02:00
|
|
|
SiteManager.site_manager.sites[self.address] = self
|
2016-09-06 14:52:55 +02:00
|
|
|
SiteManager.site_manager.load(False)
|
2016-08-10 12:35:40 +02:00
|
|
|
SiteManager.site_manager.save()
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-10-03 14:40:13 +02:00
|
|
|
def getSettingsCache(self):
|
|
|
|
back = {}
|
|
|
|
back["bad_files"] = self.bad_files
|
|
|
|
back["hashfield"] = self.content_manager.hashfield.tostring().encode("base64")
|
|
|
|
return back
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Max site size in MB
|
|
|
|
def getSizeLimit(self):
|
2016-08-16 19:55:55 +02:00
|
|
|
return self.settings.get("size_limit", int(config.size_limit))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Next size limit based on current size
|
|
|
|
def getNextSizeLimit(self):
|
|
|
|
size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
|
|
|
|
size = self.settings.get("size", 0)
|
|
|
|
for size_limit in size_limits:
|
|
|
|
if size * 1.2 < size_limit * 1024 * 1024:
|
|
|
|
return size_limit
|
|
|
|
return 999999
|
|
|
|
|
|
|
|
# Download all file from content.json
|
2016-04-06 14:01:03 +02:00
|
|
|
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
|
2015-07-12 20:36:46 +02:00
|
|
|
s = time.time()
|
2016-03-19 18:06:07 +01:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug("Downloading %s..." % inner_path)
|
2016-09-04 17:57:32 +02:00
|
|
|
|
2017-07-19 16:45:12 +02:00
|
|
|
if not inner_path.endswith("content.json"):
|
|
|
|
return False
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
content_inner_dir = helper.getDirname(inner_path)
|
2015-07-12 20:36:46 +02:00
|
|
|
if not found:
|
2015-09-17 02:20:43 +02:00
|
|
|
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
|
|
|
|
if check_modifications: # Download failed, but check modifications if its succed later
|
|
|
|
self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
|
2015-07-12 20:36:46 +02:00
|
|
|
return False # Could not download content.json
|
|
|
|
|
2016-04-06 14:01:03 +02:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug("Got %s" % inner_path)
|
2015-09-16 00:01:23 +02:00
|
|
|
changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-08-10 12:36:20 +02:00
|
|
|
if inner_path == "content.json":
|
|
|
|
self.saveSettings()
|
|
|
|
|
2016-03-30 23:05:43 +02:00
|
|
|
if peer: # Update last received update from peer to prevent re-sending the same update to it
|
|
|
|
peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"]
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Start download files
|
|
|
|
file_threads = []
|
|
|
|
if download_files:
|
|
|
|
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
|
|
|
|
file_inner_path = content_inner_dir + file_relative_path
|
2016-04-06 14:01:03 +02:00
|
|
|
|
|
|
|
# Try to diff first
|
|
|
|
diff_success = False
|
|
|
|
diff_actions = diffs.get(file_relative_path)
|
|
|
|
if diff_actions and self.bad_files.get(file_inner_path):
|
|
|
|
try:
|
2017-08-09 14:21:59 +02:00
|
|
|
s = time.time()
|
2016-04-06 14:01:03 +02:00
|
|
|
new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions)
|
|
|
|
new_file.seek(0)
|
2017-08-09 14:21:59 +02:00
|
|
|
time_diff = time.time() - s
|
|
|
|
|
|
|
|
s = time.time()
|
2016-04-06 14:01:03 +02:00
|
|
|
diff_success = self.content_manager.verifyFile(file_inner_path, new_file)
|
2017-08-09 14:21:59 +02:00
|
|
|
time_verify = time.time() - s
|
|
|
|
|
2016-04-06 14:01:03 +02:00
|
|
|
if diff_success:
|
2017-08-09 14:21:59 +02:00
|
|
|
s = time.time()
|
2016-04-06 14:01:03 +02:00
|
|
|
new_file.seek(0)
|
|
|
|
self.storage.write(file_inner_path, new_file)
|
2017-08-09 14:21:59 +02:00
|
|
|
time_write = time.time() - s
|
|
|
|
|
|
|
|
s = time.time()
|
2016-04-06 14:01:03 +02:00
|
|
|
self.onFileDone(file_inner_path)
|
2017-08-09 14:21:59 +02:00
|
|
|
time_on_done = time.time() - s
|
|
|
|
|
|
|
|
self.log.debug(
|
|
|
|
"Patched successfully: %s (diff: %.3fs, verify: %.3fs, write: %.3fs, on_done: %.3fs)" %
|
|
|
|
(file_inner_path, time_diff, time_verify, time_write, time_on_done)
|
|
|
|
)
|
2016-04-06 14:01:03 +02:00
|
|
|
except Exception, err:
|
|
|
|
self.log.debug("Failed to patch %s: %s" % (file_inner_path, err))
|
|
|
|
diff_success = False
|
|
|
|
|
|
|
|
if not diff_success:
|
|
|
|
# Start download and dont wait for finish, return the event
|
|
|
|
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
|
|
|
|
if res is not True and res is not False: # Need downloading and file is allowed
|
|
|
|
file_threads.append(res) # Append evt
|
2015-07-12 20:36:46 +02:00
|
|
|
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
# Optionals files
|
2015-11-12 02:04:45 +01:00
|
|
|
if inner_path == "content.json":
|
|
|
|
gevent.spawn(self.updateHashfield)
|
|
|
|
|
2016-11-07 23:39:45 +01:00
|
|
|
for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
|
|
|
|
file_inner_path = content_inner_dir + file_relative_path
|
|
|
|
if file_inner_path not in changed and not self.bad_files.get(file_inner_path):
|
|
|
|
continue
|
|
|
|
if not self.isDownloadable(file_inner_path):
|
|
|
|
continue
|
|
|
|
# Start download and dont wait for finish, return the event
|
|
|
|
res = self.pooledNeedFile(
|
|
|
|
file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer
|
|
|
|
)
|
|
|
|
if res is not True and res is not False: # Need downloading and file is allowed
|
|
|
|
file_threads.append(res) # Append evt
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Wait for includes download
|
|
|
|
include_threads = []
|
|
|
|
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
|
|
|
|
file_inner_path = content_inner_dir + file_relative_path
|
|
|
|
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
|
|
|
|
include_threads.append(include_thread)
|
|
|
|
|
2016-03-19 18:06:07 +01:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
|
2015-07-12 20:36:46 +02:00
|
|
|
gevent.joinall(include_threads)
|
2016-03-19 18:06:07 +01:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug("%s: Includes download ended" % inner_path)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-09-16 00:01:23 +02:00
|
|
|
if check_modifications: # Check if every file is up-to-date
|
|
|
|
self.checkModifications(0)
|
|
|
|
|
2016-03-19 18:06:07 +01:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
|
2015-07-12 20:36:46 +02:00
|
|
|
gevent.joinall(file_threads)
|
2016-03-19 18:06:07 +01:00
|
|
|
if config.verbose:
|
2017-02-27 00:13:30 +01:00
|
|
|
self.log.debug("%s: DownloadContent ended in %.3fs" % (inner_path, time.time() - s))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-03-01 23:16:31 +01:00
|
|
|
if not self.worker_manager.tasks:
|
|
|
|
self.onComplete() # No more task trigger site complete
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
return True
|
|
|
|
|
|
|
|
# Return bad files with less than 3 retry
|
|
|
|
def getReachableBadFiles(self):
|
|
|
|
if not self.bad_files:
|
|
|
|
return False
|
|
|
|
return [bad_file for bad_file, retry in self.bad_files.iteritems() if retry < 3]
|
|
|
|
|
|
|
|
# Retry download bad files
|
2015-11-17 12:48:03 +01:00
|
|
|
def retryBadFiles(self, force=False):
|
2017-11-19 18:21:10 +01:00
|
|
|
self.checkBadFiles()
|
|
|
|
|
2016-09-04 17:54:57 +02:00
|
|
|
self.log.debug("Retry %s bad files" % len(self.bad_files))
|
|
|
|
content_inner_paths = []
|
2016-11-07 23:41:25 +01:00
|
|
|
file_inner_paths = []
|
2017-11-19 18:21:10 +01:00
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
for bad_file, tries in self.bad_files.items():
|
Rev957, Sidebar displays onion peers in graph, Sidebar display bad file retry number, Sidebar site Update/Pause/Delete, Ratelimit sidebar update, Encoded typo, Fix onion findHashId, More retry for bad files, Log file path errors, Testcase for self findhashIds, Testcase for Tor findHashId, Better Tor version parse, UiWebsocket callback on update/pause/resume/delete, Skip invalid postMessage messages
2016-03-09 00:48:57 +01:00
|
|
|
if force or random.randint(0, min(40, tries)) < 4: # Larger number tries = less likely to check every 15min
|
2016-09-04 17:54:57 +02:00
|
|
|
if bad_file.endswith("content.json"):
|
|
|
|
content_inner_paths.append(bad_file)
|
|
|
|
else:
|
2016-11-07 23:39:45 +01:00
|
|
|
file_inner_paths.append(bad_file)
|
2016-09-04 17:54:57 +02:00
|
|
|
|
|
|
|
if content_inner_paths:
|
2016-11-07 23:39:45 +01:00
|
|
|
self.pooledDownloadContent(content_inner_paths, only_if_bad=True)
|
|
|
|
|
|
|
|
if file_inner_paths:
|
|
|
|
self.pooledDownloadFile(file_inner_paths, only_if_bad=True)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-11-19 18:21:10 +01:00
|
|
|
def checkBadFiles(self):
|
|
|
|
for bad_file in self.bad_files.keys():
|
|
|
|
file_info = self.content_manager.getFileInfo(bad_file)
|
|
|
|
if bad_file.endswith("content.json"):
|
|
|
|
if file_info is False and bad_file != "content.json":
|
|
|
|
del self.bad_files[bad_file]
|
|
|
|
self.log.debug("No info for file: %s, removing from bad_files" % bad_file)
|
|
|
|
else:
|
|
|
|
if file_info is False or not file_info.get("size"):
|
|
|
|
del self.bad_files[bad_file]
|
|
|
|
self.log.debug("No info or size for file: %s, removing from bad_files" % bad_file)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Download all files of the site
|
|
|
|
@util.Noparallel(blocking=False)
|
2015-08-06 00:51:25 +02:00
|
|
|
def download(self, check_size=False, blind_includes=False):
|
2017-07-28 22:57:41 +02:00
|
|
|
if not self.connection_server:
|
|
|
|
self.log.debug("No connection server found, skipping download")
|
|
|
|
return False
|
|
|
|
|
2015-08-16 11:51:00 +02:00
|
|
|
self.log.debug(
|
|
|
|
"Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" %
|
|
|
|
(self.bad_files, check_size, blind_includes)
|
|
|
|
)
|
2015-07-12 20:36:46 +02:00
|
|
|
gevent.spawn(self.announce)
|
|
|
|
if check_size: # Check the size first
|
2015-09-27 02:08:53 +02:00
|
|
|
valid = self.downloadContent("content.json", download_files=False) # Just download content.json files
|
2015-07-12 20:36:46 +02:00
|
|
|
if not valid:
|
|
|
|
return False # Cant download content.jsons or size is not fits
|
|
|
|
|
|
|
|
# Download everything
|
2015-09-16 00:01:23 +02:00
|
|
|
valid = self.downloadContent("content.json", check_modifications=blind_includes)
|
2015-08-06 00:51:25 +02:00
|
|
|
|
2016-09-04 17:57:44 +02:00
|
|
|
self.onComplete.once(lambda: self.retryBadFiles(force=True))
|
2015-08-06 00:51:25 +02:00
|
|
|
|
|
|
|
return valid
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-11-07 23:39:45 +01:00
|
|
|
def pooledDownloadContent(self, inner_paths, pool_size=100, only_if_bad=False):
|
2016-09-04 17:57:32 +02:00
|
|
|
self.log.debug("New downloadContent pool: len: %s" % len(inner_paths))
|
|
|
|
self.worker_manager.started_task_num += len(inner_paths)
|
|
|
|
pool = gevent.pool.Pool(pool_size)
|
|
|
|
for inner_path in inner_paths:
|
2016-11-07 23:39:45 +01:00
|
|
|
if not only_if_bad or inner_path in self.bad_files:
|
|
|
|
pool.spawn(self.downloadContent, inner_path)
|
2016-09-04 17:57:32 +02:00
|
|
|
self.worker_manager.started_task_num -= 1
|
|
|
|
self.log.debug("Ended downloadContent pool len: %s" % len(inner_paths))
|
|
|
|
|
2016-11-07 23:39:45 +01:00
|
|
|
def pooledDownloadFile(self, inner_paths, pool_size=100, only_if_bad=False):
|
|
|
|
self.log.debug("New downloadFile pool: len: %s" % len(inner_paths))
|
|
|
|
self.worker_manager.started_task_num += len(inner_paths)
|
|
|
|
pool = gevent.pool.Pool(pool_size)
|
|
|
|
for inner_path in inner_paths:
|
|
|
|
if not only_if_bad or inner_path in self.bad_files:
|
|
|
|
pool.spawn(self.needFile, inner_path, update=True)
|
|
|
|
self.worker_manager.started_task_num -= 1
|
|
|
|
self.log.debug("Ended downloadFile pool len: %s" % len(inner_paths))
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Update worker, try to find client that supports listModifications command
|
|
|
|
def updater(self, peers_try, queried, since):
|
|
|
|
while 1:
|
|
|
|
if not peers_try or len(queried) >= 3: # Stop after 3 successful query
|
|
|
|
break
|
|
|
|
peer = peers_try.pop(0)
|
2017-02-27 00:08:46 +01:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug("Try to get updates from: %s Left: %s" % (peer, peers_try))
|
|
|
|
|
|
|
|
res = None
|
|
|
|
with gevent.Timeout(20, exception=False):
|
|
|
|
res = peer.listModified(since)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if not res or "modified_files" not in res:
|
|
|
|
continue # Failed query
|
|
|
|
|
|
|
|
queried.append(peer)
|
2016-09-04 17:56:33 +02:00
|
|
|
modified_contents = []
|
|
|
|
my_modified = self.content_manager.listModified(since)
|
2015-07-12 20:36:46 +02:00
|
|
|
for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we
|
2017-04-07 21:17:25 +02:00
|
|
|
has_newer = int(modified) > my_modified.get(inner_path, 0)
|
|
|
|
has_older = int(modified) < my_modified.get(inner_path, 0)
|
|
|
|
if inner_path not in self.bad_files and not self.content_manager.isArchived(inner_path, modified):
|
|
|
|
if has_newer:
|
|
|
|
# We dont have this file or we have older
|
|
|
|
modified_contents.append(inner_path)
|
|
|
|
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1
|
|
|
|
if has_older:
|
|
|
|
self.log.debug("%s client has older version of %s, publishing there..." % (peer, inner_path))
|
|
|
|
gevent.spawn(self.publisher, inner_path, [peer], [], 1)
|
2016-09-04 17:57:32 +02:00
|
|
|
if modified_contents:
|
|
|
|
self.log.debug("%s new modified file from %s" % (len(modified_contents), peer))
|
|
|
|
modified_contents.sort(key=lambda inner_path: 0 - res["modified_files"][inner_path]) # Download newest first
|
|
|
|
gevent.spawn(self.pooledDownloadContent, modified_contents)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Check modified content.json files from peers and add modified files to bad_files
|
|
|
|
# Return: Successfully queried peers [Peer, Peer...]
|
|
|
|
def checkModifications(self, since=None):
|
2017-01-29 19:02:08 +01:00
|
|
|
s = time.time()
|
2015-07-12 20:36:46 +02:00
|
|
|
peers_try = [] # Try these peers
|
|
|
|
queried = [] # Successfully queried from these peers
|
2017-02-27 00:11:02 +01:00
|
|
|
limit = 5
|
2015-07-12 20:36:46 +02:00
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Wait for peers
|
|
|
|
if not self.peers:
|
Rev571, Optional file sizes to sidebar, Download all optional files option in sidebar, Optional file number in peer stats, Delete removed or changed optional files, Auto download optional files if autodownloadoptional checked, SiteReload command, Peer use global file server if no site defined, Allow browser cache video files, Allow more keepalive connections, Gevent 1.1 ranged request bugfix, Dont sent optional files details on websocket, Remove files from workermanager tasks if no longer in bad_files, Notify local client about changes on external siteSign
2015-11-09 00:44:03 +01:00
|
|
|
self.announce()
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
for wait in range(10):
|
2015-11-12 02:04:45 +01:00
|
|
|
time.sleep(5 + wait)
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
self.log.debug("Waiting for peers...")
|
2015-11-12 02:04:45 +01:00
|
|
|
if self.peers:
|
|
|
|
break
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
|
2017-02-27 00:09:59 +01:00
|
|
|
peers_try = self.getConnectedPeers()
|
|
|
|
peers_connected_num = len(peers_try)
|
|
|
|
if peers_connected_num < limit * 2: # Add more, non-connected peers if necessary
|
|
|
|
peers_try += self.getRecentPeers(limit * 5)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
if since is None: # No since defined, download from last modification time-1day
|
2015-07-12 20:36:46 +02:00
|
|
|
since = self.settings.get("modified", 60 * 60 * 24) - 60 * 60 * 24
|
2017-06-19 16:12:41 +02:00
|
|
|
|
|
|
|
if config.verbose:
|
|
|
|
self.log.debug(
|
|
|
|
"Try to get listModifications from peers: %s, connected: %s, since: %s" %
|
|
|
|
(peers_try, peers_connected_num, since)
|
|
|
|
)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
updaters = []
|
|
|
|
for i in range(3):
|
|
|
|
updaters.append(gevent.spawn(self.updater, peers_try, queried, since))
|
|
|
|
|
2015-08-16 11:51:00 +02:00
|
|
|
gevent.joinall(updaters, timeout=10) # Wait 10 sec to workers done query modifications
|
2015-11-02 22:39:19 +01:00
|
|
|
|
2017-02-27 00:09:41 +01:00
|
|
|
if not queried: # Start another 3 thread if first 3 is stuck
|
2017-05-23 12:40:42 +02:00
|
|
|
peers_try[0:0] = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Add connected peers
|
2017-02-27 00:09:41 +01:00
|
|
|
for _ in range(10):
|
|
|
|
gevent.joinall(updaters, timeout=10) # Wait another 10 sec if none of updaters finished
|
|
|
|
if queried:
|
|
|
|
break
|
|
|
|
|
|
|
|
self.log.debug("Queried listModifications from: %s in %.3fs" % (queried, time.time() - s))
|
2015-07-12 20:36:46 +02:00
|
|
|
time.sleep(0.1)
|
|
|
|
return queried
|
|
|
|
|
|
|
|
# Update content.json from peers and download changed files
|
|
|
|
# Return: None
|
|
|
|
@util.Noparallel()
|
2017-01-29 19:02:08 +01:00
|
|
|
def update(self, announce=False, check_files=False, since=None):
|
2016-08-10 12:41:04 +02:00
|
|
|
self.content_manager.loadContent("content.json", load_includes=False) # Reload content.json
|
2015-07-12 20:36:46 +02:00
|
|
|
self.content_updated = None # Reset content updated time
|
|
|
|
self.updateWebsocket(updating=True)
|
2016-09-17 21:00:20 +02:00
|
|
|
|
2017-01-29 19:03:08 +01:00
|
|
|
# Remove files that no longer in content.json
|
2017-11-19 18:21:10 +01:00
|
|
|
self.checkBadFiles()
|
2016-09-17 21:00:20 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if announce:
|
|
|
|
self.announce()
|
|
|
|
|
2017-01-29 19:03:22 +01:00
|
|
|
# Full update, we can reset bad files
|
|
|
|
if check_files and since == 0:
|
|
|
|
self.bad_files = {}
|
|
|
|
|
2017-01-29 19:02:08 +01:00
|
|
|
queried = self.checkModifications(since)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-03-18 19:19:16 +01:00
|
|
|
if check_files:
|
2016-09-04 17:59:29 +02:00
|
|
|
self.storage.updateBadFiles(quick_check=True) # Quick check and mark bad files based on file size
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-08-10 12:41:04 +02:00
|
|
|
changed, deleted = self.content_manager.loadContent("content.json", load_includes=False)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
if self.bad_files:
|
2015-11-17 12:48:03 +01:00
|
|
|
self.log.debug("Bad files: %s" % self.bad_files)
|
2016-11-07 23:41:25 +01:00
|
|
|
gevent.spawn(self.retryBadFiles, force=True)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-03-18 19:18:59 +01:00
|
|
|
if len(queried) == 0:
|
|
|
|
# Failed to query modifications
|
|
|
|
self.content_updated = False
|
|
|
|
self.bad_files["content.json"] = 1
|
2018-01-19 02:29:18 +01:00
|
|
|
else:
|
|
|
|
self.content_updated = time.time()
|
2016-03-18 19:18:59 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
self.updateWebsocket(updated=True)
|
|
|
|
|
|
|
|
# Update site by redownload all content.json
|
|
|
|
def redownloadContents(self):
|
|
|
|
# Download all content.json again
|
|
|
|
content_threads = []
|
|
|
|
for inner_path in self.content_manager.contents.keys():
|
|
|
|
content_threads.append(self.needFile(inner_path, update=True, blocking=False))
|
|
|
|
|
|
|
|
self.log.debug("Waiting %s content.json to finish..." % len(content_threads))
|
|
|
|
gevent.joinall(content_threads)
|
|
|
|
|
|
|
|
# Publish worker
|
2017-01-16 13:41:57 +01:00
|
|
|
def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None):
|
2015-07-12 20:36:46 +02:00
|
|
|
file_size = self.storage.getSize(inner_path)
|
2016-03-30 23:05:43 +02:00
|
|
|
content_json_modified = self.content_manager.contents[inner_path]["modified"]
|
2015-07-12 20:36:46 +02:00
|
|
|
body = self.storage.read(inner_path)
|
2016-03-26 01:33:37 +01:00
|
|
|
|
|
|
|
# Find out my ip and port
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
tor_manager = self.connection_server.tor_manager
|
2016-04-06 13:59:53 +02:00
|
|
|
if tor_manager and tor_manager.enabled and tor_manager.start_onions:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
my_ip = tor_manager.getOnion(self.address)
|
|
|
|
if my_ip:
|
|
|
|
my_ip += ".onion"
|
|
|
|
my_port = config.fileserver_port
|
|
|
|
else:
|
|
|
|
my_ip = config.ip_external
|
|
|
|
if self.connection_server.port_opened:
|
|
|
|
my_port = config.fileserver_port
|
|
|
|
else:
|
|
|
|
my_port = 0
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
while 1:
|
|
|
|
if not peers or len(published) >= limit:
|
|
|
|
if event_done:
|
|
|
|
event_done.set(True)
|
|
|
|
break # All peers done, or published engouht
|
|
|
|
peer = peers.pop(0)
|
2016-04-06 14:00:40 +02:00
|
|
|
if peer in peers: # Remove duplicate
|
|
|
|
peers.remove(peer)
|
2016-03-26 01:33:37 +01:00
|
|
|
if peer in published:
|
|
|
|
continue
|
2016-03-30 23:05:43 +02:00
|
|
|
if peer.last_content_json_update == content_json_modified:
|
|
|
|
self.log.debug("%s already received this update for %s, skipping" % (peer, inner_path))
|
|
|
|
continue
|
2016-03-26 01:33:37 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if peer.connection and peer.connection.last_ping_delay: # Peer connected
|
|
|
|
# Timeout: 5sec + size in kb + last_ping
|
2016-03-16 00:39:09 +01:00
|
|
|
timeout = 5 + int(file_size / 1024) + peer.connection.last_ping_delay
|
2015-07-12 20:36:46 +02:00
|
|
|
else: # Peer not connected
|
2016-03-16 00:39:09 +01:00
|
|
|
# Timeout: 10sec + size in kb
|
|
|
|
timeout = 10 + int(file_size / 1024)
|
2015-07-12 20:36:46 +02:00
|
|
|
result = {"exception": "Timeout"}
|
|
|
|
|
|
|
|
for retry in range(2):
|
|
|
|
try:
|
|
|
|
with gevent.Timeout(timeout, False):
|
|
|
|
result = peer.request("update", {
|
|
|
|
"site": self.address,
|
|
|
|
"inner_path": inner_path,
|
|
|
|
"body": body,
|
2017-11-19 18:21:47 +01:00
|
|
|
"diffs": diffs
|
2015-07-12 20:36:46 +02:00
|
|
|
})
|
|
|
|
if result:
|
|
|
|
break
|
|
|
|
except Exception, err:
|
2016-04-06 13:59:53 +02:00
|
|
|
self.log.error("Publish error: %s" % Debug.formatException(err))
|
2015-07-12 20:36:46 +02:00
|
|
|
result = {"exception": Debug.formatException(err)}
|
|
|
|
|
|
|
|
if result and "ok" in result:
|
|
|
|
published.append(peer)
|
2017-01-16 13:41:57 +01:00
|
|
|
if cb_progress and len(published) <= limit:
|
|
|
|
cb_progress(len(published), limit)
|
2016-03-03 21:12:16 +01:00
|
|
|
self.log.info("[OK] %s: %s %s/%s" % (peer.key, result["ok"], len(published), limit))
|
2015-07-12 20:36:46 +02:00
|
|
|
else:
|
|
|
|
if result == {"exception": "Timeout"}:
|
2017-02-27 00:02:24 +01:00
|
|
|
peer.onConnectionError("Publish timeout")
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.info("[FAILED] %s: %s" % (peer.key, result))
|
2016-04-06 13:59:53 +02:00
|
|
|
time.sleep(0.01)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Update content.json on peers
|
|
|
|
@util.Noparallel()
|
2017-01-16 13:41:57 +01:00
|
|
|
def publish(self, limit="default", inner_path="content.json", diffs={}, cb_progress=None):
|
2015-07-12 20:36:46 +02:00
|
|
|
published = [] # Successfully published (Peer)
|
|
|
|
publishers = [] # Publisher threads
|
2015-09-08 03:07:44 +02:00
|
|
|
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
if not self.peers:
|
|
|
|
self.announce()
|
|
|
|
|
2016-03-03 21:12:16 +01:00
|
|
|
if limit == "default":
|
2017-04-07 18:17:30 +02:00
|
|
|
limit = 5
|
2016-03-26 01:33:37 +01:00
|
|
|
threads = limit
|
2016-03-03 21:12:16 +01:00
|
|
|
|
2016-03-16 00:40:01 +01:00
|
|
|
peers = self.getConnectedPeers()
|
|
|
|
num_connected_peers = len(peers)
|
|
|
|
|
|
|
|
random.shuffle(peers)
|
2016-09-17 21:00:40 +02:00
|
|
|
peers = sorted(peers, key=lambda peer: peer.connection.handshake.get("rev", 0) < config.rev - 100) # Prefer newer clients
|
2016-03-16 00:40:01 +01:00
|
|
|
|
2017-02-27 00:11:02 +01:00
|
|
|
if len(peers) < limit * 2: # Add more, non-connected peers if necessary
|
|
|
|
peers += self.getRecentPeers(limit * 2)
|
2015-09-08 03:07:44 +02:00
|
|
|
|
2016-04-10 23:12:34 +02:00
|
|
|
self.log.info("Publishing %s to %s/%s peers (connected: %s) diffs: %s (%.2fk)..." % (
|
2016-04-20 23:30:25 +02:00
|
|
|
inner_path, limit, len(self.peers), num_connected_peers, diffs.keys(), float(len(str(diffs))) / 1024
|
2015-09-08 03:07:44 +02:00
|
|
|
))
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if not peers:
|
|
|
|
return 0 # No peers found
|
|
|
|
|
|
|
|
event_done = gevent.event.AsyncResult()
|
2016-04-06 13:57:19 +02:00
|
|
|
for i in range(min(len(peers), limit, threads)):
|
2017-01-16 13:41:57 +01:00
|
|
|
publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit, diffs, event_done, cb_progress)
|
2015-07-12 20:36:46 +02:00
|
|
|
publishers.append(publisher)
|
|
|
|
|
|
|
|
event_done.get() # Wait for done
|
|
|
|
if len(published) < min(len(self.peers), limit):
|
|
|
|
time.sleep(0.2) # If less than we need sleep a bit
|
|
|
|
if len(published) == 0:
|
|
|
|
gevent.joinall(publishers) # No successful publish, wait for all publisher
|
|
|
|
|
2016-03-16 21:11:04 +01:00
|
|
|
# Publish more peers in the backgroup
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.info(
|
2016-04-20 23:30:25 +02:00
|
|
|
"Successfuly %s published to %s peers, publishing to %s more peers in the background" %
|
|
|
|
(inner_path, len(published), limit)
|
|
|
|
)
|
2015-09-08 03:07:44 +02:00
|
|
|
|
2016-03-16 21:11:04 +01:00
|
|
|
for thread in range(2):
|
2016-10-02 14:22:32 +02:00
|
|
|
gevent.spawn(self.publisher, inner_path, peers, published, limit=limit * 2, diffs=diffs)
|
2015-09-08 03:07:44 +02:00
|
|
|
|
2015-11-02 22:39:19 +01:00
|
|
|
# Send my hashfield to every connected peer if changed
|
|
|
|
gevent.spawn(self.sendMyHashfield, 100)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
return len(published)
|
|
|
|
|
|
|
|
# Copy this site
|
2016-12-08 17:14:55 +01:00
|
|
|
def clone(self, address, privatekey=None, address_index=None, root_inner_path="", overwrite=False):
|
2015-07-12 20:36:46 +02:00
|
|
|
import shutil
|
|
|
|
new_site = SiteManager.site_manager.need(address, all_file=False)
|
|
|
|
default_dirs = [] # Dont copy these directories (has -default version)
|
|
|
|
for dir_name in os.listdir(self.storage.directory):
|
|
|
|
if "-default" in dir_name:
|
|
|
|
default_dirs.append(dir_name.replace("-default", ""))
|
|
|
|
|
2016-12-08 17:14:55 +01:00
|
|
|
self.log.debug("Cloning to %s, ignore dirs: %s, root: %s" % (address, default_dirs, root_inner_path))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Copy root content.json
|
|
|
|
if not new_site.storage.isFile("content.json") and not overwrite:
|
2017-12-04 16:04:10 +01:00
|
|
|
# New site: Content.json not exist yet, create a new one from source site
|
|
|
|
if "size_limit" in self.settings:
|
|
|
|
new_site.settings["size_limit"] = self.settings["size_limit"]
|
|
|
|
|
|
|
|
# Use content.json-default is specified
|
2016-12-14 00:42:31 +01:00
|
|
|
if self.storage.isFile(root_inner_path + "/content.json-default"):
|
|
|
|
content_json = self.storage.loadJson(root_inner_path + "/content.json-default")
|
|
|
|
else:
|
|
|
|
content_json = self.storage.loadJson("content.json")
|
2017-12-04 16:04:10 +01:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
if "domain" in content_json:
|
|
|
|
del content_json["domain"]
|
|
|
|
content_json["title"] = "my" + content_json["title"]
|
|
|
|
content_json["cloned_from"] = self.address
|
2016-12-08 17:14:55 +01:00
|
|
|
content_json["clone_root"] = root_inner_path
|
2016-09-04 18:00:51 +02:00
|
|
|
content_json["files"] = {}
|
2015-07-12 20:36:46 +02:00
|
|
|
if address_index:
|
|
|
|
content_json["address_index"] = address_index # Site owner's BIP32 index
|
|
|
|
new_site.storage.writeJson("content.json", content_json)
|
2015-09-16 00:01:23 +02:00
|
|
|
new_site.content_manager.loadContent(
|
|
|
|
"content.json", add_bad_files=False, delete_removed_files=False, load_includes=False
|
|
|
|
)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Copy files
|
|
|
|
for content_inner_path, content in self.content_manager.contents.items():
|
2017-06-19 15:39:14 +02:00
|
|
|
file_relative_paths = content.get("files", {}).keys()
|
|
|
|
|
|
|
|
# Sign content.json at the end to make sure every file is included
|
|
|
|
file_relative_paths.sort()
|
|
|
|
file_relative_paths.sort(key=lambda key: key.replace("-default", "").endswith("content.json"))
|
|
|
|
|
|
|
|
for file_relative_path in file_relative_paths:
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
file_inner_path = helper.getDirname(content_inner_path) + file_relative_path # Relative to content.json
|
2015-07-12 20:36:46 +02:00
|
|
|
file_inner_path = file_inner_path.strip("/") # Strip leading /
|
2016-12-08 17:14:55 +01:00
|
|
|
if not file_inner_path.startswith(root_inner_path):
|
|
|
|
self.log.debug("[SKIP] %s (not in clone root)" % file_inner_path)
|
|
|
|
continue
|
2015-07-12 20:36:46 +02:00
|
|
|
if file_inner_path.split("/")[0] in default_dirs: # Dont copy directories that has -default postfixed alternative
|
|
|
|
self.log.debug("[SKIP] %s (has default alternative)" % file_inner_path)
|
|
|
|
continue
|
|
|
|
file_path = self.storage.getPath(file_inner_path)
|
|
|
|
|
|
|
|
# Copy the file normally to keep the -default postfixed dir and file to allow cloning later
|
2016-12-08 17:14:55 +01:00
|
|
|
if root_inner_path:
|
2016-12-14 00:42:52 +01:00
|
|
|
file_inner_path_dest = re.sub("^%s/" % re.escape(root_inner_path), "", file_inner_path)
|
2016-12-08 17:14:55 +01:00
|
|
|
file_path_dest = new_site.storage.getPath(file_inner_path_dest)
|
|
|
|
else:
|
2016-12-14 00:42:52 +01:00
|
|
|
file_inner_path_dest = file_inner_path
|
2016-12-08 17:14:55 +01:00
|
|
|
file_path_dest = new_site.storage.getPath(file_inner_path)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug("[COPY] %s to %s..." % (file_inner_path, file_path_dest))
|
|
|
|
dest_dir = os.path.dirname(file_path_dest)
|
|
|
|
if not os.path.isdir(dest_dir):
|
|
|
|
os.makedirs(dest_dir)
|
2017-06-19 15:39:14 +02:00
|
|
|
if file_inner_path_dest.replace("-default", "") == "content.json": # Don't copy root content.json-default
|
2016-12-14 00:43:08 +01:00
|
|
|
continue
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
shutil.copy(file_path, file_path_dest)
|
|
|
|
|
|
|
|
# If -default in path, create a -default less copy of the file
|
|
|
|
if "-default" in file_inner_path:
|
|
|
|
file_path_dest = new_site.storage.getPath(file_inner_path.replace("-default", ""))
|
2016-04-20 23:30:25 +02:00
|
|
|
if new_site.storage.isFile(file_inner_path.replace("-default", "")) and not overwrite:
|
|
|
|
# Don't overwrite site files with default ones
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug("[SKIP] Default file: %s (already exist)" % file_inner_path)
|
|
|
|
continue
|
|
|
|
self.log.debug("[COPY] Default file: %s to %s..." % (file_inner_path, file_path_dest))
|
|
|
|
dest_dir = os.path.dirname(file_path_dest)
|
|
|
|
if not os.path.isdir(dest_dir):
|
|
|
|
os.makedirs(dest_dir)
|
|
|
|
shutil.copy(file_path, file_path_dest)
|
|
|
|
# Sign if content json
|
|
|
|
if file_path_dest.endswith("/content.json"):
|
|
|
|
new_site.storage.onUpdated(file_inner_path.replace("-default", ""))
|
|
|
|
new_site.content_manager.loadContent(
|
2015-09-16 00:01:23 +02:00
|
|
|
file_inner_path.replace("-default", ""), add_bad_files=False,
|
|
|
|
delete_removed_files=False, load_includes=False
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
|
|
|
if privatekey:
|
|
|
|
new_site.content_manager.sign(file_inner_path.replace("-default", ""), privatekey)
|
2015-09-17 02:20:43 +02:00
|
|
|
new_site.content_manager.loadContent(
|
|
|
|
file_inner_path, add_bad_files=False, delete_removed_files=False, load_includes=False
|
|
|
|
)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
if privatekey:
|
|
|
|
new_site.content_manager.sign("content.json", privatekey)
|
2015-09-17 02:20:43 +02:00
|
|
|
new_site.content_manager.loadContent(
|
|
|
|
"content.json", add_bad_files=False, delete_removed_files=False, load_includes=False
|
|
|
|
)
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Rebuild DB
|
|
|
|
if new_site.storage.isFile("dbschema.json"):
|
2015-09-24 22:08:08 +02:00
|
|
|
new_site.storage.closeDb()
|
2015-07-12 20:36:46 +02:00
|
|
|
new_site.storage.rebuildDb()
|
|
|
|
|
|
|
|
return new_site
|
|
|
|
|
2016-11-07 23:39:45 +01:00
|
|
|
@util.Pooled(100)
|
|
|
|
def pooledNeedFile(self, *args, **kwargs):
|
|
|
|
return self.needFile(*args, **kwargs)
|
|
|
|
|
2017-10-03 15:29:26 +02:00
|
|
|
def isFileDownloadAllowed(self, inner_path, file_info):
|
|
|
|
if file_info.get("size", 0) > config.file_size_limit * 1024 * 1024:
|
|
|
|
self.log.debug(
|
|
|
|
"File size %s too large: %sMB > %sMB, skipping..." %
|
|
|
|
(inner_path, file_info.get("size", 0) / 1024 / 1024, config.file_size_limit)
|
|
|
|
)
|
|
|
|
return False
|
|
|
|
else:
|
|
|
|
return True
|
|
|
|
|
2017-10-03 15:28:37 +02:00
|
|
|
def needFileInfo(self, inner_path):
|
|
|
|
file_info = self.content_manager.getFileInfo(inner_path)
|
|
|
|
if not file_info:
|
|
|
|
# No info for file, download all content.json first
|
|
|
|
self.log.debug("No info for %s, waiting for all content.json" % inner_path)
|
|
|
|
success = self.downloadContent("content.json", download_files=False)
|
|
|
|
if not success:
|
|
|
|
return False
|
|
|
|
file_info = self.content_manager.getFileInfo(inner_path)
|
|
|
|
return file_info
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# Check and download if file not exist
|
|
|
|
def needFile(self, inner_path, update=False, blocking=True, peer=None, priority=0):
|
|
|
|
if self.storage.isFile(inner_path) and not update: # File exist, no need to do anything
|
|
|
|
return True
|
|
|
|
elif self.settings["serving"] is False: # Site not serving
|
|
|
|
return False
|
|
|
|
else: # Wait until file downloaded
|
2015-08-16 11:51:00 +02:00
|
|
|
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file
|
2015-07-12 20:36:46 +02:00
|
|
|
if not self.content_manager.contents.get("content.json"): # No content.json, download it first!
|
|
|
|
self.log.debug("Need content.json first")
|
|
|
|
gevent.spawn(self.announce)
|
|
|
|
if inner_path != "content.json": # Prevent double download
|
|
|
|
task = self.worker_manager.addTask("content.json", peer)
|
2017-10-03 15:47:46 +02:00
|
|
|
task["evt"].get()
|
2015-07-12 20:36:46 +02:00
|
|
|
self.content_manager.loadContent()
|
|
|
|
if not self.content_manager.contents.get("content.json"):
|
|
|
|
return False # Content.json download failed
|
|
|
|
|
2017-10-03 15:28:37 +02:00
|
|
|
file_info = None
|
2016-08-10 12:41:35 +02:00
|
|
|
if not inner_path.endswith("content.json"):
|
2017-10-03 15:28:37 +02:00
|
|
|
file_info = self.needFileInfo(inner_path)
|
2016-08-10 12:41:35 +02:00
|
|
|
if not file_info:
|
2017-10-03 15:28:37 +02:00
|
|
|
return False
|
2016-08-10 12:41:35 +02:00
|
|
|
if "cert_signers" in file_info and not file_info["content_inner_path"] in self.content_manager.contents:
|
|
|
|
self.log.debug("Missing content.json for requested user file: %s" % inner_path)
|
2016-08-16 19:56:18 +02:00
|
|
|
if self.bad_files.get(file_info["content_inner_path"], 0) > 5:
|
|
|
|
self.log.debug("File %s not reachable: retry %s" % (
|
|
|
|
inner_path, self.bad_files.get(file_info["content_inner_path"], 0)
|
|
|
|
))
|
|
|
|
return False
|
2016-09-04 18:01:09 +02:00
|
|
|
self.downloadContent(file_info["content_inner_path"])
|
2017-10-03 15:29:26 +02:00
|
|
|
|
|
|
|
if not self.isFileDownloadAllowed(inner_path, file_info):
|
|
|
|
self.log.debug("%s: Download not allowed" % inner_path)
|
2017-05-23 12:41:37 +02:00
|
|
|
return False
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-10-03 15:49:08 +02:00
|
|
|
task = self.worker_manager.addTask(inner_path, peer, priority=priority, file_info=file_info)
|
2015-07-12 20:36:46 +02:00
|
|
|
if blocking:
|
2017-10-03 15:47:46 +02:00
|
|
|
return task["evt"].get()
|
2015-07-12 20:36:46 +02:00
|
|
|
else:
|
2017-10-03 15:49:08 +02:00
|
|
|
return task["evt"]
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Add or update a peer to site
|
2015-09-28 00:22:27 +02:00
|
|
|
# return_peer: Always return the peer even if it was already present
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
def addPeer(self, ip, port, return_peer=False, connection=None):
|
2017-12-15 12:28:48 +01:00
|
|
|
if not ip or ip == "0.0.0.0":
|
2015-07-12 20:36:46 +02:00
|
|
|
return False
|
|
|
|
key = "%s:%s" % (ip, port)
|
|
|
|
if key in self.peers: # Already has this ip
|
2015-07-25 13:38:58 +02:00
|
|
|
self.peers[key].found()
|
2015-07-12 20:36:46 +02:00
|
|
|
if return_peer: # Always return peer
|
|
|
|
return self.peers[key]
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
else: # New peer
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if (ip, port) in self.peer_blacklist:
|
|
|
|
return False # Ignore blacklist (eg. myself)
|
2015-07-12 20:36:46 +02:00
|
|
|
peer = Peer(ip, port, self)
|
|
|
|
self.peers[key] = peer
|
|
|
|
return peer
|
|
|
|
|
|
|
|
# Gather peer from connected peers
|
|
|
|
@util.Noparallel(blocking=False)
|
|
|
|
def announcePex(self, query_num=2, need_num=5):
|
|
|
|
peers = [peer for peer in self.peers.values() if peer.connection and peer.connection.connected] # Connected peers
|
|
|
|
if len(peers) == 0: # Small number of connected peers for this site, connect to any
|
|
|
|
self.log.debug("Small number of peers detected...query all of peers using pex")
|
|
|
|
peers = self.peers.values()
|
|
|
|
need_num = 10
|
|
|
|
|
|
|
|
random.shuffle(peers)
|
|
|
|
done = 0
|
|
|
|
added = 0
|
|
|
|
for peer in peers:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
res = peer.pex(need_num=need_num)
|
2015-07-12 20:36:46 +02:00
|
|
|
if type(res) == int: # We have result
|
|
|
|
done += 1
|
|
|
|
added += res
|
|
|
|
if res:
|
|
|
|
self.worker_manager.onPeers()
|
|
|
|
self.updateWebsocket(peers_added=res)
|
|
|
|
if done == query_num:
|
|
|
|
break
|
|
|
|
self.log.debug("Queried pex from %s peers got %s new peers." % (done, added))
|
|
|
|
|
|
|
|
# Gather peers from tracker
|
|
|
|
# Return: Complete time or False on error
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
def announceTracker(self, tracker_protocol, tracker_address, fileserver_port=0, add_types=[], my_peer_id="", mode="start"):
|
2015-07-12 20:36:46 +02:00
|
|
|
s = time.time()
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if "ip4" not in add_types:
|
|
|
|
fileserver_port = 0
|
|
|
|
|
|
|
|
if tracker_protocol == "udp": # Udp tracker
|
2015-07-12 20:36:46 +02:00
|
|
|
if config.disable_udp:
|
|
|
|
return False # No udp supported
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
ip, port = tracker_address.split(":")
|
2015-09-10 23:25:09 +02:00
|
|
|
tracker = UdpTrackerClient(ip, int(port))
|
2015-07-12 20:36:46 +02:00
|
|
|
tracker.peer_port = fileserver_port
|
|
|
|
try:
|
|
|
|
tracker.connect()
|
|
|
|
tracker.poll_once()
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
tracker.announce(info_hash=hashlib.sha1(self.address).hexdigest(), num_want=50)
|
2015-07-12 20:36:46 +02:00
|
|
|
back = tracker.poll_once()
|
|
|
|
peers = back["response"]["peers"]
|
|
|
|
except Exception, err:
|
|
|
|
return False
|
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
elif tracker_protocol == "http": # Http tracker
|
2015-07-12 20:36:46 +02:00
|
|
|
params = {
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
'info_hash': hashlib.sha1(self.address).digest(),
|
2015-07-12 20:36:46 +02:00
|
|
|
'peer_id': my_peer_id, 'port': fileserver_port,
|
|
|
|
'uploaded': 0, 'downloaded': 0, 'left': 0, 'compact': 1, 'numwant': 30,
|
|
|
|
'event': 'started'
|
|
|
|
}
|
|
|
|
req = None
|
|
|
|
try:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
url = "http://" + tracker_address + "?" + urllib.urlencode(params)
|
2015-07-12 20:36:46 +02:00
|
|
|
# Load url
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
with gevent.Timeout(30, False): # Make sure of timeout
|
|
|
|
req = urllib2.urlopen(url, timeout=25)
|
2015-07-12 20:36:46 +02:00
|
|
|
response = req.read()
|
|
|
|
req.fp._sock.recv = None # Hacky avoidance of memory leak for older python versions
|
|
|
|
req.close()
|
|
|
|
req = None
|
|
|
|
if not response:
|
2017-06-19 16:12:24 +02:00
|
|
|
self.log.debug("Http tracker %s response error" % tracker_address)
|
2015-07-12 20:36:46 +02:00
|
|
|
return False
|
|
|
|
# Decode peers
|
|
|
|
peer_data = bencode.decode(response)["peers"]
|
|
|
|
response = None
|
|
|
|
peer_count = len(peer_data) / 6
|
|
|
|
peers = []
|
|
|
|
for peer_offset in xrange(peer_count):
|
|
|
|
off = 6 * peer_offset
|
|
|
|
peer = peer_data[off:off + 6]
|
|
|
|
addr, port = struct.unpack('!LH', peer)
|
|
|
|
peers.append({"addr": socket.inet_ntoa(struct.pack('!L', addr)), "port": port})
|
|
|
|
except Exception, err:
|
2017-06-19 16:12:24 +02:00
|
|
|
self.log.debug("Http tracker %s error: %s" % (tracker_address, err))
|
2015-07-12 20:36:46 +02:00
|
|
|
if req:
|
|
|
|
req.close()
|
|
|
|
req = None
|
|
|
|
return False
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
else:
|
|
|
|
peers = []
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Adding peers
|
|
|
|
added = 0
|
|
|
|
for peer in peers:
|
|
|
|
if not peer["port"]:
|
|
|
|
continue # Dont add peers with port 0
|
|
|
|
if self.addPeer(peer["addr"], peer["port"]):
|
|
|
|
added += 1
|
|
|
|
if added:
|
|
|
|
self.worker_manager.onPeers()
|
|
|
|
self.updateWebsocket(peers_added=added)
|
2017-12-15 12:10:36 +01:00
|
|
|
self.log.debug("%s: Found %s peers, new: %s, total: %s" % (tracker_address, len(peers), added, len(self.peers)))
|
2015-07-12 20:36:46 +02:00
|
|
|
return time.time() - s
|
|
|
|
|
|
|
|
# Add myself and get other peers from tracker
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
def announce(self, force=False, mode="start", pex=True):
|
2015-10-22 11:42:55 +02:00
|
|
|
if time.time() < self.time_announce + 30 and not force:
|
2015-07-12 20:36:46 +02:00
|
|
|
return # No reannouncing within 30 secs
|
2015-10-22 11:42:55 +02:00
|
|
|
self.time_announce = time.time()
|
2015-09-10 23:25:09 +02:00
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
trackers = config.trackers
|
|
|
|
# Filter trackers based on supported networks
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
if config.disable_udp:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
trackers = [tracker for tracker in trackers if not tracker.startswith("udp://")]
|
2016-11-07 23:42:00 +01:00
|
|
|
if self.connection_server and self.connection_server.tor_manager and not self.connection_server.tor_manager.enabled:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
trackers = [tracker for tracker in trackers if ".onion" not in tracker]
|
|
|
|
|
2016-11-07 23:42:00 +01:00
|
|
|
if trackers and (mode == "update" or mode == "more"): # Only announce on one tracker, increment the queried tracker id
|
2015-09-10 23:25:09 +02:00
|
|
|
self.last_tracker_id += 1
|
|
|
|
self.last_tracker_id = self.last_tracker_id % len(trackers)
|
|
|
|
trackers = [trackers[self.last_tracker_id]] # We only going to use this one
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
errors = []
|
|
|
|
slow = []
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
add_types = []
|
|
|
|
if self.connection_server:
|
|
|
|
my_peer_id = self.connection_server.peer_id
|
|
|
|
|
|
|
|
# Type of addresses they can reach me
|
|
|
|
if self.connection_server.port_opened:
|
|
|
|
add_types.append("ip4")
|
2016-11-07 23:42:00 +01:00
|
|
|
if self.connection_server.tor_manager and self.connection_server.tor_manager.start_onions:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
add_types.append("onion")
|
|
|
|
else:
|
|
|
|
my_peer_id = ""
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
s = time.time()
|
|
|
|
announced = 0
|
|
|
|
threads = []
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
fileserver_port = config.fileserver_port
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2015-09-10 23:25:09 +02:00
|
|
|
for tracker in trackers: # Start announce threads
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
tracker_protocol, tracker_address = tracker.split("://")
|
|
|
|
thread = gevent.spawn(
|
|
|
|
self.announceTracker, tracker_protocol, tracker_address, fileserver_port, add_types, my_peer_id, mode
|
|
|
|
)
|
2015-07-12 20:36:46 +02:00
|
|
|
threads.append(thread)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
thread.tracker_address = tracker_address
|
|
|
|
thread.tracker_protocol = tracker_protocol
|
2015-07-12 20:36:46 +02:00
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
gevent.joinall(threads, timeout=10) # Wait for announce finish
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
for thread in threads:
|
|
|
|
if thread.value:
|
|
|
|
if thread.value > 1:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
slow.append("%.2fs %s://%s" % (thread.value, thread.tracker_protocol, thread.tracker_address))
|
2015-07-12 20:36:46 +02:00
|
|
|
announced += 1
|
|
|
|
else:
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
if thread.ready():
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
errors.append("%s://%s" % (thread.tracker_protocol, thread.tracker_address))
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
else: # Still running
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
slow.append("10s+ %s://%s" % (thread.tracker_protocol, thread.tracker_address))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
# Save peers num
|
|
|
|
self.settings["peers"] = len(self.peers)
|
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if len(errors) < len(threads): # Less errors than total tracker nums
|
2018-01-19 02:27:19 +01:00
|
|
|
if announced == 1:
|
|
|
|
announced_to = trackers[0]
|
|
|
|
else:
|
|
|
|
announced_to = "%s trackers" % announced
|
2015-07-12 20:36:46 +02:00
|
|
|
self.log.debug(
|
2018-01-19 02:27:19 +01:00
|
|
|
"Announced types %s in mode %s to %s in %.3fs, errors: %s, slow: %s" %
|
|
|
|
(add_types, mode, announced_to, time.time() - s, errors, slow)
|
2015-07-12 20:36:46 +02:00
|
|
|
)
|
|
|
|
else:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if mode != "update":
|
2015-11-17 12:48:03 +01:00
|
|
|
self.log.error("Announce to %s trackers in %.3fs, failed" % (announced, time.time() - s))
|
2015-09-10 23:25:09 +02:00
|
|
|
|
|
|
|
if pex:
|
|
|
|
if not [peer for peer in self.peers.values() if peer.connection and peer.connection.connected]:
|
|
|
|
# If no connected peer yet then wait for connections
|
|
|
|
gevent.spawn_later(3, self.announcePex, need_num=10) # Spawn 3 secs later
|
|
|
|
else: # Else announce immediately
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
if mode == "more": # Need more peers
|
|
|
|
self.announcePex(need_num=10)
|
|
|
|
else:
|
|
|
|
self.announcePex()
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2016-04-29 02:58:00 +02:00
|
|
|
# Keep connections to get the updates
|
2017-10-21 11:02:40 +02:00
|
|
|
def needConnections(self, num=6, check_site_on_reconnect=False):
|
2016-08-26 11:42:23 +02:00
|
|
|
need = min(len(self.peers), num, config.connected_limit) # Need 5 peer, but max total peers
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-02-24 01:16:22 +01:00
|
|
|
connected = len(self.getConnectedPeers())
|
2015-07-12 20:36:46 +02:00
|
|
|
|
2017-02-27 00:10:27 +01:00
|
|
|
connected_before = connected
|
|
|
|
|
2017-02-24 01:16:22 +01:00
|
|
|
self.log.debug("Need connections: %s, Current: %s, Total: %s" % (need, connected, len(self.peers)))
|
2015-07-12 20:36:46 +02:00
|
|
|
|
|
|
|
if connected < need: # Need more than we have
|
|
|
|
for peer in self.peers.values():
|
|
|
|
if not peer.connection or not peer.connection.connected: # No peer connection or disconnected
|
|
|
|
peer.pex() # Initiate peer exchange
|
|
|
|
if peer.connection and peer.connection.connected:
|
|
|
|
connected += 1 # Successfully connected
|
|
|
|
if connected >= need:
|
|
|
|
break
|
2017-05-23 12:40:42 +02:00
|
|
|
self.log.debug(
|
|
|
|
"Connected before: %s, after: %s. Check site: %s." %
|
|
|
|
(connected_before, connected, check_site_on_reconnect)
|
|
|
|
)
|
2017-02-27 00:10:27 +01:00
|
|
|
|
|
|
|
if check_site_on_reconnect and connected_before == 0 and connected > 0 and self.connection_server.has_internet:
|
|
|
|
gevent.spawn(self.update, check_files=False)
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
return connected
|
|
|
|
|
2017-02-27 00:13:18 +01:00
|
|
|
# Return: Probably peers verified to be connectable recently
|
2015-07-12 20:36:46 +02:00
|
|
|
def getConnectablePeers(self, need_num=5, ignore=[]):
|
|
|
|
peers = self.peers.values()
|
|
|
|
found = []
|
|
|
|
for peer in peers:
|
|
|
|
if peer.key.endswith(":0"):
|
|
|
|
continue # Not connectable
|
|
|
|
if not peer.connection:
|
|
|
|
continue # No connection
|
|
|
|
if peer.key in ignore:
|
|
|
|
continue # The requester has this peer
|
|
|
|
if time.time() - peer.connection.last_recv_time > 60 * 60 * 2: # Last message more than 2 hours ago
|
|
|
|
peer.connection = None # Cleanup: Dead connection
|
|
|
|
continue
|
|
|
|
found.append(peer)
|
|
|
|
if len(found) >= need_num:
|
|
|
|
break # Found requested number of peers
|
|
|
|
|
2017-12-19 16:14:13 +01:00
|
|
|
if len(found) < need_num: # Return not that good peers
|
2015-07-12 20:36:46 +02:00
|
|
|
found = [peer for peer in peers if not peer.key.endswith(":0") and peer.key not in ignore][0:need_num - len(found)]
|
|
|
|
|
|
|
|
return found
|
|
|
|
|
2017-02-27 00:11:45 +01:00
|
|
|
# Return: Recently found peers
|
|
|
|
def getRecentPeers(self, need_num):
|
2017-05-23 12:40:42 +02:00
|
|
|
found = sorted(
|
|
|
|
self.peers.values()[0:need_num * 50],
|
|
|
|
key=lambda peer: peer.time_found + peer.reputation * 60,
|
|
|
|
reverse=True
|
|
|
|
)[0:need_num * 2]
|
2017-02-27 00:11:45 +01:00
|
|
|
random.shuffle(found)
|
|
|
|
return found[0:need_num]
|
|
|
|
|
2015-09-08 03:07:44 +02:00
|
|
|
def getConnectedPeers(self):
|
2017-02-27 00:11:57 +01:00
|
|
|
back = []
|
2018-01-19 02:27:40 +01:00
|
|
|
if not self.connection_server:
|
|
|
|
return []
|
|
|
|
|
2017-10-21 11:02:27 +02:00
|
|
|
tor_manager = self.connection_server.tor_manager
|
2017-02-27 00:11:57 +01:00
|
|
|
for connection in self.connection_server.connections:
|
|
|
|
if not connection.connected and time.time() - connection.start_time > 20: # Still not connected after 20s
|
|
|
|
continue
|
|
|
|
peer = self.peers.get("%s:%s" % (connection.ip, connection.port))
|
|
|
|
if peer:
|
2017-10-21 11:02:27 +02:00
|
|
|
if connection.target_onion and tor_manager.start_onions and tor_manager.getOnion(self.address) != connection.target_onion:
|
|
|
|
continue
|
2017-02-27 00:11:57 +01:00
|
|
|
if not peer.connection:
|
|
|
|
peer.connect(connection)
|
|
|
|
back.append(peer)
|
|
|
|
return back
|
2015-09-08 03:07:44 +02:00
|
|
|
|
2016-03-03 21:12:16 +01:00
|
|
|
# Cleanup probably dead peers and close connection if too much
|
2017-05-07 18:15:23 +02:00
|
|
|
def cleanupPeers(self, peers_protected=[]):
|
2015-07-25 13:38:58 +02:00
|
|
|
peers = self.peers.values()
|
2016-08-26 11:42:23 +02:00
|
|
|
if len(peers) > 20:
|
|
|
|
# Cleanup old peers
|
|
|
|
removed = 0
|
2016-11-07 23:40:04 +01:00
|
|
|
if len(peers) > 1000:
|
|
|
|
ttl = 60 * 60 * 1
|
|
|
|
else:
|
|
|
|
ttl = 60 * 60 * 4
|
2015-07-25 13:38:58 +02:00
|
|
|
|
2016-08-26 11:42:23 +02:00
|
|
|
for peer in peers:
|
|
|
|
if peer.connection and peer.connection.connected:
|
|
|
|
continue
|
|
|
|
if peer.connection and not peer.connection.connected:
|
|
|
|
peer.connection = None # Dead connection
|
2016-11-07 23:40:04 +01:00
|
|
|
if time.time() - peer.time_found > ttl: # Not found on tracker or via pex in last 4 hour
|
2017-02-27 00:02:24 +01:00
|
|
|
peer.remove("Time found expired")
|
2016-08-26 11:42:23 +02:00
|
|
|
removed += 1
|
|
|
|
if removed > len(peers) * 0.1: # Don't remove too much at once
|
|
|
|
break
|
2015-07-25 13:38:58 +02:00
|
|
|
|
2016-08-26 11:42:23 +02:00
|
|
|
if removed:
|
|
|
|
self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers)))
|
2015-07-25 13:38:58 +02:00
|
|
|
|
2016-03-16 21:11:34 +01:00
|
|
|
# Close peers over the limit
|
2016-03-03 21:12:16 +01:00
|
|
|
closed = 0
|
2017-02-27 00:12:36 +01:00
|
|
|
connected_peers = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Only fully connected peers
|
2016-03-03 21:12:16 +01:00
|
|
|
need_to_close = len(connected_peers) - config.connected_limit
|
|
|
|
|
|
|
|
if closed < need_to_close:
|
2017-05-23 12:40:42 +02:00
|
|
|
# Try to keep connections with more sites
|
|
|
|
for peer in sorted(connected_peers, key=lambda peer: min(peer.connection.sites, 5)):
|
2017-02-27 00:12:55 +01:00
|
|
|
if not peer.connection:
|
|
|
|
continue
|
2017-05-07 18:15:23 +02:00
|
|
|
if peer.key in peers_protected:
|
|
|
|
continue
|
2017-02-27 00:12:55 +01:00
|
|
|
if peer.connection.sites > 5:
|
|
|
|
break
|
|
|
|
peer.connection.close("Cleanup peers")
|
|
|
|
peer.connection = None
|
2016-03-03 21:12:16 +01:00
|
|
|
closed += 1
|
|
|
|
if closed >= need_to_close:
|
|
|
|
break
|
|
|
|
|
|
|
|
if need_to_close > 0:
|
|
|
|
self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % (len(connected_peers), need_to_close, closed))
|
|
|
|
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
# Send hashfield to peers
|
2016-11-07 23:42:13 +01:00
|
|
|
def sendMyHashfield(self, limit=5):
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
if not self.content_manager.hashfield: # No optional files
|
|
|
|
return False
|
2015-11-12 02:04:45 +01:00
|
|
|
|
|
|
|
sent = 0
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
connected_peers = self.getConnectedPeers()
|
|
|
|
for peer in connected_peers:
|
|
|
|
if peer.sendMyHashfield():
|
2015-11-12 02:04:45 +01:00
|
|
|
sent += 1
|
|
|
|
if sent >= limit:
|
2015-11-02 22:39:19 +01:00
|
|
|
break
|
2015-11-12 02:04:45 +01:00
|
|
|
if sent:
|
2018-01-19 02:30:52 +01:00
|
|
|
my_hashfield_changed = self.content_manager.hashfield.time_changed
|
|
|
|
self.log.debug("Sent my hashfield (chaged %.3fs ago) to %s peers" % (time.time() - my_hashfield_changed, sent))
|
2015-11-12 02:04:45 +01:00
|
|
|
return sent
|
|
|
|
|
|
|
|
# Update hashfield
|
2016-11-07 23:42:13 +01:00
|
|
|
def updateHashfield(self, limit=5):
|
2015-11-12 02:04:45 +01:00
|
|
|
# Return if no optional files
|
2015-11-17 12:48:03 +01:00
|
|
|
if not self.content_manager.hashfield and not self.content_manager.contents.get("content.json", {}).get("files_optional"):
|
2015-11-12 02:04:45 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
queried = 0
|
|
|
|
connected_peers = self.getConnectedPeers()
|
|
|
|
for peer in connected_peers:
|
|
|
|
if peer.time_hashfield:
|
|
|
|
continue
|
|
|
|
if peer.updateHashfield():
|
|
|
|
queried += 1
|
|
|
|
if queried >= limit:
|
|
|
|
break
|
|
|
|
if queried:
|
|
|
|
self.log.debug("Queried hashfield from %s peers" % queried)
|
|
|
|
return queried
|
Rev536, Fix stats page, Support ranged http requests for better video browser compatibility, setHashfield command, One by one send hashfield to connected peers if changed, Keep count hashfield changetime, PeerHashfield optimalizations, Wait for peers on checkmodification, Give more time to query trackers, Do not count udp trackers as error if udp disabled, Test hashfield push
2015-10-30 02:08:02 +01:00
|
|
|
|
2016-11-07 23:39:04 +01:00
|
|
|
# Returns if the optional file is need to be downloaded or not
|
|
|
|
def isDownloadable(self, inner_path):
|
|
|
|
return self.settings.get("autodownloadoptional")
|
|
|
|
|
2016-09-05 13:55:51 +02:00
|
|
|
def delete(self):
|
|
|
|
self.settings["serving"] = False
|
|
|
|
self.saveSettings()
|
|
|
|
self.worker_manager.running = False
|
|
|
|
self.worker_manager.stopWorkers()
|
|
|
|
self.storage.deleteFiles()
|
|
|
|
self.updateWebsocket()
|
2016-11-07 23:42:28 +01:00
|
|
|
self.content_manager.contents.db.deleteSite(self)
|
2016-09-06 10:16:09 +02:00
|
|
|
SiteManager.site_manager.delete(self.address)
|
2016-09-05 13:55:51 +02:00
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# - Events -
|
|
|
|
|
|
|
|
# Add event listeners
|
|
|
|
def addEventListeners(self):
|
|
|
|
self.onFileStart = util.Event() # If WorkerManager added new task
|
2015-07-25 13:38:58 +02:00
|
|
|
self.onFileDone = util.Event() # If WorkerManager successfully downloaded a file
|
2015-07-12 20:36:46 +02:00
|
|
|
self.onFileFail = util.Event() # If WorkerManager failed to download a file
|
|
|
|
self.onComplete = util.Event() # All file finished
|
|
|
|
|
|
|
|
self.onFileStart.append(lambda inner_path: self.fileStarted()) # No parameters to make Noparallel batching working
|
|
|
|
self.onFileDone.append(lambda inner_path: self.fileDone(inner_path))
|
|
|
|
self.onFileFail.append(lambda inner_path: self.fileFailed(inner_path))
|
|
|
|
|
|
|
|
# Send site status update to websocket clients
|
|
|
|
def updateWebsocket(self, **kwargs):
|
|
|
|
if kwargs:
|
|
|
|
param = {"event": kwargs.items()[0]}
|
|
|
|
else:
|
|
|
|
param = None
|
|
|
|
for ws in self.websockets:
|
|
|
|
ws.event("siteChanged", self, param)
|
|
|
|
|
2017-05-07 18:14:13 +02:00
|
|
|
def messageWebsocket(self, message, type="info", progress=None):
|
|
|
|
for ws in self.websockets:
|
|
|
|
if progress is None:
|
|
|
|
ws.cmd("notification", [type, message])
|
|
|
|
else:
|
|
|
|
ws.cmd("progress", [type, message, progress])
|
|
|
|
|
2015-07-12 20:36:46 +02:00
|
|
|
# File download started
|
|
|
|
@util.Noparallel(blocking=False)
|
|
|
|
def fileStarted(self):
|
|
|
|
time.sleep(0.001) # Wait for other files adds
|
|
|
|
self.updateWebsocket(file_started=True)
|
|
|
|
|
|
|
|
# File downloaded successful
|
|
|
|
def fileDone(self, inner_path):
|
|
|
|
# File downloaded, remove it from bad files
|
|
|
|
if inner_path in self.bad_files:
|
2016-04-06 13:39:17 +02:00
|
|
|
if config.verbose:
|
|
|
|
self.log.debug("Bad file solved: %s" % inner_path)
|
2015-07-12 20:36:46 +02:00
|
|
|
del(self.bad_files[inner_path])
|
|
|
|
|
|
|
|
# Update content.json last downlad time
|
|
|
|
if inner_path == "content.json":
|
|
|
|
self.content_updated = time.time()
|
|
|
|
|
|
|
|
self.updateWebsocket(file_done=inner_path)
|
|
|
|
|
|
|
|
# File download failed
|
|
|
|
def fileFailed(self, inner_path):
|
|
|
|
if inner_path == "content.json":
|
|
|
|
self.content_updated = False
|
|
|
|
self.log.debug("Can't update content.json")
|
2017-02-19 00:52:20 +01:00
|
|
|
if inner_path in self.bad_files and self.connection_server.has_internet:
|
2015-07-12 20:36:46 +02:00
|
|
|
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1
|
|
|
|
|
|
|
|
self.updateWebsocket(file_failed=inner_path)
|
2017-02-19 00:52:31 +01:00
|
|
|
|
2017-02-19 12:32:41 +01:00
|
|
|
if self.bad_files.get(inner_path, 0) > 30:
|
2017-02-22 11:28:17 +01:00
|
|
|
self.log.debug("Giving up on %s" % inner_path)
|
2017-02-19 00:52:31 +01:00
|
|
|
del self.bad_files[inner_path] # Give up after 30 tries
|