2015-09-16 01:04:37 +02:00
|
|
|
import os
|
2016-05-26 15:59:40 +02:00
|
|
|
import stat
|
2015-09-27 02:08:53 +02:00
|
|
|
import socket
|
|
|
|
import struct
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
import re
|
2015-10-22 11:42:55 +02:00
|
|
|
import collections
|
2015-11-05 23:19:36 +01:00
|
|
|
import time
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
import logging
|
|
|
|
import base64
|
2019-08-02 14:05:14 +02:00
|
|
|
import json
|
|
|
|
|
2016-11-07 22:50:45 +01:00
|
|
|
import gevent
|
|
|
|
|
|
|
|
from Config import config
|
2015-09-16 01:04:37 +02:00
|
|
|
|
2015-09-21 20:09:32 +02:00
|
|
|
|
2019-03-15 23:49:55 +01:00
|
|
|
def atomicWrite(dest, content, mode="wb"):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
try:
|
2017-01-05 03:03:17 +01:00
|
|
|
with open(dest + "-tmpnew", mode) as f:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
f.write(content)
|
|
|
|
f.flush()
|
|
|
|
os.fsync(f.fileno())
|
2017-01-05 03:03:17 +01:00
|
|
|
if os.path.isfile(dest + "-tmpold"): # Previous incomplete write
|
|
|
|
os.rename(dest + "-tmpold", dest + "-tmpold-%s" % time.time())
|
2018-12-13 21:17:36 +01:00
|
|
|
if os.path.isfile(dest): # Rename old file to -tmpold
|
|
|
|
os.rename(dest, dest + "-tmpold")
|
2017-01-05 03:03:17 +01:00
|
|
|
os.rename(dest + "-tmpnew", dest)
|
2019-03-27 03:12:02 +01:00
|
|
|
if os.path.isfile(dest + "-tmpold"):
|
|
|
|
os.unlink(dest + "-tmpold") # Remove old file
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
return True
|
2019-03-15 23:50:33 +01:00
|
|
|
except Exception as err:
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
from Debug import Debug
|
|
|
|
logging.error(
|
2019-03-15 23:50:33 +01:00
|
|
|
"File %s write failed: %s, (%s) reverting..." %
|
|
|
|
(dest, Debug.formatException(err), Debug.formatStack())
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
)
|
2017-01-05 03:03:17 +01:00
|
|
|
if os.path.isfile(dest + "-tmpold") and not os.path.isfile(dest):
|
|
|
|
os.rename(dest + "-tmpold", dest)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
|
2019-08-02 14:05:14 +02:00
|
|
|
def jsonDumps(data):
|
|
|
|
content = json.dumps(data, indent=1, sort_keys=True)
|
|
|
|
|
|
|
|
# Make it a little more compact by removing unnecessary white space
|
|
|
|
def compact_dict(match):
|
|
|
|
if "\n" in match.group(0):
|
|
|
|
return match.group(0).replace(match.group(1), match.group(1).strip())
|
|
|
|
else:
|
|
|
|
return match.group(0)
|
|
|
|
|
2019-10-06 03:10:20 +02:00
|
|
|
content = re.sub(r"\{(\n[^,\[\{]{10,100000}?)\}[, ]{0,2}\n", compact_dict, content, flags=re.DOTALL)
|
2019-08-02 14:05:14 +02:00
|
|
|
|
|
|
|
def compact_list(match):
|
|
|
|
if "\n" in match.group(0):
|
|
|
|
stripped_lines = re.sub("\n[ ]*", "", match.group(1))
|
|
|
|
return match.group(0).replace(match.group(1), stripped_lines)
|
|
|
|
else:
|
|
|
|
return match.group(0)
|
|
|
|
|
2019-10-06 03:10:20 +02:00
|
|
|
content = re.sub(r"\[([^\[\{]{2,100000}?)\][, ]{0,2}\n", compact_list, content, flags=re.DOTALL)
|
2019-08-02 14:05:14 +02:00
|
|
|
|
|
|
|
# Remove end of line whitespace
|
|
|
|
content = re.sub(r"(?m)[ ]+$", "", content)
|
|
|
|
return content
|
|
|
|
|
|
|
|
|
2019-03-15 23:53:48 +01:00
|
|
|
def openLocked(path, mode="wb"):
|
2019-03-20 01:05:52 +01:00
|
|
|
try:
|
|
|
|
if os.name == "posix":
|
|
|
|
import fcntl
|
|
|
|
f = open(path, mode)
|
|
|
|
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
|
|
elif os.name == "nt":
|
|
|
|
import msvcrt
|
|
|
|
f = open(path, mode)
|
|
|
|
msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1)
|
|
|
|
else:
|
|
|
|
f = open(path, mode)
|
|
|
|
except (IOError, PermissionError, BlockingIOError) as err:
|
|
|
|
raise BlockingIOError("Unable to lock file: %s" % err)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
return f
|
2015-09-21 20:09:32 +02:00
|
|
|
|
|
|
|
|
2016-11-07 22:50:01 +01:00
|
|
|
def getFreeSpace():
|
|
|
|
free_space = -1
|
|
|
|
if "statvfs" in dir(os): # Unix
|
2017-01-23 12:55:03 +01:00
|
|
|
statvfs = os.statvfs(config.data_dir.encode("utf8"))
|
2016-11-07 22:50:01 +01:00
|
|
|
free_space = statvfs.f_frsize * statvfs.f_bavail
|
|
|
|
else: # Windows
|
|
|
|
try:
|
|
|
|
import ctypes
|
|
|
|
free_space_pointer = ctypes.c_ulonglong(0)
|
|
|
|
ctypes.windll.kernel32.GetDiskFreeSpaceExW(
|
|
|
|
ctypes.c_wchar_p(config.data_dir), None, None, ctypes.pointer(free_space_pointer)
|
|
|
|
)
|
|
|
|
free_space = free_space_pointer.value
|
2019-03-15 21:06:59 +01:00
|
|
|
except Exception as err:
|
2016-11-07 22:50:01 +01:00
|
|
|
logging.error("GetFreeSpace error: %s" % err)
|
|
|
|
return free_space
|
|
|
|
|
|
|
|
|
2019-04-29 17:18:02 +02:00
|
|
|
def sqlquote(value):
|
|
|
|
if type(value) is int:
|
|
|
|
return str(value)
|
|
|
|
else:
|
|
|
|
return "'%s'" % value.replace("'", "''")
|
|
|
|
|
|
|
|
|
2015-09-21 20:09:32 +02:00
|
|
|
def shellquote(*args):
|
|
|
|
if len(args) == 1:
|
|
|
|
return '"%s"' % args[0].replace('"', "")
|
|
|
|
else:
|
|
|
|
return tuple(['"%s"' % arg.replace('"', "") for arg in args])
|
2015-09-27 02:08:53 +02:00
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
def packPeers(peers):
|
2019-01-20 03:21:07 +01:00
|
|
|
packed_peers = {"ipv4": [], "ipv6": [], "onion": []}
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
for peer in peers:
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
try:
|
2019-01-20 03:21:07 +01:00
|
|
|
ip_type = getIpType(peer.ip)
|
2019-12-19 02:16:41 +01:00
|
|
|
if ip_type in packed_peers:
|
|
|
|
packed_peers[ip_type].append(peer.packMyAddress())
|
2016-11-07 22:50:33 +01:00
|
|
|
except Exception:
|
2019-12-17 15:00:23 +01:00
|
|
|
logging.debug("Error packing peer address: %s" % peer)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
return packed_peers
|
|
|
|
|
|
|
|
|
2019-01-20 03:21:07 +01:00
|
|
|
# ip, port to packed 6byte or 18byte format
|
2015-09-27 02:08:53 +02:00
|
|
|
def packAddress(ip, port):
|
2019-01-20 03:21:07 +01:00
|
|
|
if ":" in ip:
|
|
|
|
return socket.inet_pton(socket.AF_INET6, ip) + struct.pack("H", port)
|
|
|
|
else:
|
|
|
|
return socket.inet_aton(ip) + struct.pack("H", port)
|
2015-09-27 02:08:53 +02:00
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
2019-01-20 03:21:07 +01:00
|
|
|
# From 6byte or 18byte format to ip, port
|
2015-09-27 02:08:53 +02:00
|
|
|
def unpackAddress(packed):
|
2019-01-20 03:21:07 +01:00
|
|
|
if len(packed) == 18:
|
|
|
|
return socket.inet_ntop(socket.AF_INET6, packed[0:16]), struct.unpack_from("H", packed, 16)[0]
|
|
|
|
else:
|
2019-07-03 18:35:55 +02:00
|
|
|
if len(packed) != 6:
|
|
|
|
raise Exception("Invalid length ip4 packed address: %s" % len(packed))
|
2019-01-20 03:21:07 +01:00
|
|
|
return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# onion, port to packed 12byte format
|
|
|
|
def packOnionAddress(onion, port):
|
|
|
|
onion = onion.replace(".onion", "")
|
|
|
|
return base64.b32decode(onion.upper()) + struct.pack("H", port)
|
|
|
|
|
|
|
|
|
|
|
|
# From 12byte format to ip, port
|
|
|
|
def unpackOnionAddress(packed):
|
2019-03-15 23:55:40 +01:00
|
|
|
return base64.b32encode(packed[0:-2]).lower().decode() + ".onion", struct.unpack("H", packed[-2:])[0]
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
# Get dir from file
|
2017-08-18 14:37:56 +02:00
|
|
|
# Return: data/site/content.json -> data/site/
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
def getDirname(path):
|
2016-04-06 13:48:13 +02:00
|
|
|
if "/" in path:
|
2017-10-03 16:05:01 +02:00
|
|
|
return path[:path.rfind("/") + 1].lstrip("/")
|
2016-04-06 13:48:13 +02:00
|
|
|
else:
|
|
|
|
return ""
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
2016-11-07 22:50:33 +01:00
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
# Get dir from file
|
|
|
|
# Return: data/site/content.json -> content.json
|
|
|
|
def getFilename(path):
|
2016-11-07 22:50:33 +01:00
|
|
|
return path[path.rfind("/") + 1:]
|
2015-10-22 11:42:55 +02:00
|
|
|
|
2019-01-20 03:21:07 +01:00
|
|
|
|
2017-10-04 12:44:34 +02:00
|
|
|
def getFilesize(path):
|
|
|
|
try:
|
|
|
|
s = os.stat(path)
|
2019-08-02 14:04:18 +02:00
|
|
|
except Exception:
|
2017-10-04 12:44:34 +02:00
|
|
|
return None
|
|
|
|
if stat.S_ISREG(s.st_mode): # Test if it's file
|
|
|
|
return s.st_size
|
|
|
|
else:
|
|
|
|
return None
|
2015-10-22 11:42:55 +02:00
|
|
|
|
2019-01-20 03:21:38 +01:00
|
|
|
|
2015-10-22 11:42:55 +02:00
|
|
|
# Convert hash to hashid for hashfield
|
|
|
|
def toHashId(hash):
|
|
|
|
return int(hash[0:4], 16)
|
|
|
|
|
|
|
|
|
|
|
|
# Merge dict values
|
|
|
|
def mergeDicts(dicts):
|
|
|
|
back = collections.defaultdict(set)
|
|
|
|
for d in dicts:
|
2019-03-15 21:06:59 +01:00
|
|
|
for key, val in d.items():
|
2015-10-22 11:42:55 +02:00
|
|
|
back[key].update(val)
|
|
|
|
return dict(back)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Request https url using gevent SSL error workaround
|
|
|
|
def httpRequest(url, as_file=False):
|
|
|
|
if url.startswith("http://"):
|
2019-03-15 21:06:59 +01:00
|
|
|
import urllib.request
|
|
|
|
response = urllib.request.urlopen(url)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
else: # Hack to avoid Python gevent ssl errors
|
|
|
|
import socket
|
2019-03-15 21:06:59 +01:00
|
|
|
import http.client
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
import ssl
|
|
|
|
|
|
|
|
host, request = re.match("https://(.*?)(/.*?)$", url).groups()
|
|
|
|
|
2019-03-15 21:06:59 +01:00
|
|
|
conn = http.client.HTTPSConnection(host)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
sock = socket.create_connection((conn.host, conn.port), conn.timeout, conn.source_address)
|
|
|
|
conn.sock = ssl.wrap_socket(sock, conn.key_file, conn.cert_file)
|
|
|
|
conn.request("GET", request)
|
|
|
|
response = conn.getresponse()
|
2016-03-16 22:07:11 +01:00
|
|
|
if response.status in [301, 302, 303, 307, 308]:
|
|
|
|
logging.info("Redirect to: %s" % response.getheader('Location'))
|
|
|
|
response = httpRequest(response.getheader('Location'))
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
|
|
|
if as_file:
|
2019-03-15 21:06:59 +01:00
|
|
|
import io
|
|
|
|
data = io.BytesIO()
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
while True:
|
|
|
|
buff = response.read(1024 * 16)
|
|
|
|
if not buff:
|
|
|
|
break
|
|
|
|
data.write(buff)
|
|
|
|
return data
|
|
|
|
else:
|
|
|
|
return response
|
2016-11-07 22:50:45 +01:00
|
|
|
|
|
|
|
|
|
|
|
def timerCaller(secs, func, *args, **kwargs):
|
|
|
|
gevent.spawn_later(secs, timerCaller, secs, func, *args, **kwargs)
|
|
|
|
func(*args, **kwargs)
|
|
|
|
|
|
|
|
|
|
|
|
def timer(secs, func, *args, **kwargs):
|
2019-12-17 15:00:09 +01:00
|
|
|
return gevent.spawn_later(secs, timerCaller, secs, func, *args, **kwargs)
|
2017-04-14 16:05:42 +02:00
|
|
|
|
|
|
|
|
|
|
|
def create_connection(address, timeout=None, source_address=None):
|
|
|
|
if address in config.ip_local:
|
|
|
|
sock = socket.create_connection_original(address, timeout, source_address)
|
|
|
|
else:
|
|
|
|
sock = socket.create_connection_original(address, timeout, socket.bind_addr)
|
|
|
|
return sock
|
|
|
|
|
2019-01-20 03:21:38 +01:00
|
|
|
|
2017-04-14 16:05:42 +02:00
|
|
|
def socketBindMonkeyPatch(bind_ip, bind_port):
|
|
|
|
import socket
|
|
|
|
logging.info("Monkey patching socket to bind to: %s:%s" % (bind_ip, bind_port))
|
|
|
|
socket.bind_addr = (bind_ip, int(bind_port))
|
|
|
|
socket.create_connection_original = socket.create_connection
|
|
|
|
socket.create_connection = create_connection
|
2017-08-09 14:19:09 +02:00
|
|
|
|
|
|
|
|
|
|
|
def limitedGzipFile(*args, **kwargs):
|
|
|
|
import gzip
|
2019-01-20 03:21:38 +01:00
|
|
|
|
2017-08-09 14:19:09 +02:00
|
|
|
class LimitedGzipFile(gzip.GzipFile):
|
|
|
|
def read(self, size=-1):
|
2019-01-20 03:21:38 +01:00
|
|
|
return super(LimitedGzipFile, self).read(1024 * 1024 * 25)
|
2017-08-09 14:19:09 +02:00
|
|
|
return LimitedGzipFile(*args, **kwargs)
|
2018-01-19 02:21:54 +01:00
|
|
|
|
2019-01-20 03:21:38 +01:00
|
|
|
|
2018-01-19 02:21:54 +01:00
|
|
|
def avg(items):
|
|
|
|
if len(items) > 0:
|
|
|
|
return sum(items) / len(items)
|
|
|
|
else:
|
2018-01-30 13:58:01 +01:00
|
|
|
return 0
|
|
|
|
|
2019-01-20 03:21:38 +01:00
|
|
|
|
2018-12-19 13:15:59 +01:00
|
|
|
def isIp(ip):
|
|
|
|
if ":" in ip: # IPv6
|
|
|
|
try:
|
|
|
|
socket.inet_pton(socket.AF_INET6, ip)
|
|
|
|
return True
|
2019-08-02 14:04:18 +02:00
|
|
|
except Exception:
|
2018-12-19 13:15:59 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
else: # IPv4
|
|
|
|
try:
|
|
|
|
socket.inet_aton(ip)
|
|
|
|
return True
|
2019-08-02 14:04:18 +02:00
|
|
|
except Exception:
|
2018-12-19 13:15:59 +01:00
|
|
|
return False
|
|
|
|
|
2019-01-20 03:21:38 +01:00
|
|
|
|
2019-01-29 03:00:06 +01:00
|
|
|
local_ip_pattern = re.compile(r"^127\.|192\.168\.|10\.|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[0-1]\.|169\.254\.|::1$|fe80")
|
2018-01-30 13:58:01 +01:00
|
|
|
def isPrivateIp(ip):
|
|
|
|
return local_ip_pattern.match(ip)
|
2019-01-20 03:21:27 +01:00
|
|
|
|
|
|
|
|
|
|
|
def getIpType(ip):
|
|
|
|
if ip.endswith(".onion"):
|
|
|
|
return "onion"
|
|
|
|
elif ":" in ip:
|
|
|
|
return "ipv6"
|
2019-12-19 02:16:41 +01:00
|
|
|
elif re.match("[0-9\.]+$", ip):
|
2019-01-20 03:21:27 +01:00
|
|
|
return "ipv4"
|
2019-12-19 02:16:41 +01:00
|
|
|
else:
|
|
|
|
return "unknown"
|
2019-01-20 03:21:27 +01:00
|
|
|
|
|
|
|
|
2019-01-29 02:59:15 +01:00
|
|
|
def createSocket(ip, sock_type=socket.SOCK_STREAM):
|
2019-01-20 03:21:27 +01:00
|
|
|
ip_type = getIpType(ip)
|
|
|
|
if ip_type == "ipv6":
|
2019-01-29 02:59:15 +01:00
|
|
|
return socket.socket(socket.AF_INET6, sock_type)
|
2019-01-20 03:21:27 +01:00
|
|
|
else:
|
2019-01-29 02:59:15 +01:00
|
|
|
return socket.socket(socket.AF_INET, sock_type)
|
|
|
|
|
|
|
|
|
|
|
|
def getInterfaceIps(ip_type="ipv4"):
|
|
|
|
res = []
|
|
|
|
if ip_type == "ipv6":
|
|
|
|
test_ips = ["ff0e::c", "2606:4700:4700::1111"]
|
|
|
|
else:
|
|
|
|
test_ips = ['239.255.255.250', "8.8.8.8"]
|
|
|
|
|
|
|
|
for test_ip in test_ips:
|
|
|
|
try:
|
|
|
|
s = createSocket(test_ip, sock_type=socket.SOCK_DGRAM)
|
|
|
|
s.connect((test_ip, 1))
|
|
|
|
res.append(s.getsockname()[0])
|
2019-08-02 14:04:18 +02:00
|
|
|
except Exception:
|
2019-01-29 02:59:15 +01:00
|
|
|
pass
|
|
|
|
|
|
|
|
try:
|
|
|
|
res += [ip[4][0] for ip in socket.getaddrinfo(socket.gethostname(), 1)]
|
2019-08-02 14:04:18 +02:00
|
|
|
except Exception:
|
2019-01-29 02:59:15 +01:00
|
|
|
pass
|
|
|
|
|
2019-01-30 14:51:45 +01:00
|
|
|
res = [re.sub("%.*", "", ip) for ip in res if getIpType(ip) == ip_type and isIp(ip)]
|
2019-01-29 02:59:15 +01:00
|
|
|
return list(set(res))
|
2019-03-15 23:55:23 +01:00
|
|
|
|
|
|
|
|
|
|
|
def cmp(a, b):
|
|
|
|
return (a > b) - (a < b)
|
|
|
|
|
|
|
|
|
|
|
|
def encodeResponse(func):
|
|
|
|
def wrapper(*args, **kwargs):
|
|
|
|
back = func(*args, **kwargs)
|
|
|
|
if "__next__" in dir(back):
|
|
|
|
for part in back:
|
2019-10-06 03:10:43 +02:00
|
|
|
if type(part) == bytes:
|
|
|
|
yield part
|
|
|
|
else:
|
|
|
|
yield part.encode()
|
2019-03-15 23:55:23 +01:00
|
|
|
else:
|
2019-10-06 03:10:43 +02:00
|
|
|
if type(back) == bytes:
|
|
|
|
yield back
|
|
|
|
else:
|
|
|
|
yield back.encode()
|
2019-03-15 23:55:23 +01:00
|
|
|
|
|
|
|
return wrapper
|