2015-09-16 01:04:37 +02:00
|
|
|
import os
|
2016-05-26 15:59:40 +02:00
|
|
|
import stat
|
2015-09-27 02:08:53 +02:00
|
|
|
import socket
|
|
|
|
import struct
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
import re
|
2015-10-22 11:42:55 +02:00
|
|
|
import collections
|
2015-11-05 23:19:36 +01:00
|
|
|
import time
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
import logging
|
|
|
|
import base64
|
2015-09-16 01:04:37 +02:00
|
|
|
|
2015-09-21 20:09:32 +02:00
|
|
|
|
2015-09-16 01:04:37 +02:00
|
|
|
def atomicWrite(dest, content, mode="w"):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
try:
|
2016-05-26 15:59:40 +02:00
|
|
|
permissions = stat.S_IMODE(os.lstat(dest).st_mode)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
with open(dest + "-new", mode) as f:
|
|
|
|
f.write(content)
|
|
|
|
f.flush()
|
|
|
|
os.fsync(f.fileno())
|
|
|
|
if os.path.isfile(dest + "-old"): # Previous incomplete write
|
|
|
|
os.rename(dest + "-old", dest + "-old-%s" % time.time())
|
|
|
|
os.rename(dest, dest + "-old")
|
|
|
|
os.rename(dest + "-new", dest)
|
2016-05-26 15:59:40 +02:00
|
|
|
os.chmod(dest, permissions)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
os.unlink(dest + "-old")
|
|
|
|
return True
|
|
|
|
except Exception, err:
|
|
|
|
from Debug import Debug
|
|
|
|
logging.error(
|
|
|
|
"File %s write failed: %s, reverting..." %
|
|
|
|
(dest, Debug.formatException(err))
|
|
|
|
)
|
|
|
|
if os.path.isfile(dest + "-old") and not os.path.isfile(dest):
|
|
|
|
os.rename(dest + "-old", dest)
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
def openLocked(path, mode="w"):
|
|
|
|
if os.name == "posix":
|
|
|
|
import fcntl
|
|
|
|
f = open(path, mode)
|
|
|
|
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
2016-11-07 22:34:12 +01:00
|
|
|
elif os.name == "nt":
|
|
|
|
import msvcrt
|
|
|
|
f = open(path, mode)
|
|
|
|
msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, -1)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
else:
|
|
|
|
f = open(path, mode)
|
|
|
|
return f
|
2015-09-21 20:09:32 +02:00
|
|
|
|
|
|
|
|
|
|
|
def shellquote(*args):
|
|
|
|
if len(args) == 1:
|
|
|
|
return '"%s"' % args[0].replace('"', "")
|
|
|
|
else:
|
|
|
|
return tuple(['"%s"' % arg.replace('"', "") for arg in args])
|
2015-09-27 02:08:53 +02:00
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
def packPeers(peers):
|
|
|
|
packed_peers = {"ip4": [], "onion": []}
|
|
|
|
for peer in peers:
|
Version 0.3.6, Rev879, Fix sidebar error on description missing, New trayicon, New favicon, Disable some functions on MultiUser proxies, New homepage, Replace only the last ? in SQL queries, Alwaays grant ADMIN permission to homepage site, Announce before publish if no peers, configSet, serverShutdown, ADMIN WebsocketAPI command, Stop Tor client before updating, Ignore peer ip packing error, Ignore db files from git, Fix safari ajax error when UiPassword enabled
2016-02-02 11:40:45 +01:00
|
|
|
try:
|
|
|
|
if peer.ip.endswith(".onion"):
|
|
|
|
packed_peers["onion"].append(peer.packMyAddress())
|
|
|
|
else:
|
|
|
|
packed_peers["ip4"].append(peer.packMyAddress())
|
|
|
|
except Exception, err:
|
|
|
|
logging.error("Error packing peer address: %s" % peer)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
return packed_peers
|
|
|
|
|
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
# ip, port to packed 6byte format
|
|
|
|
def packAddress(ip, port):
|
|
|
|
return socket.inet_aton(ip) + struct.pack("H", port)
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
2015-09-27 02:08:53 +02:00
|
|
|
# From 6byte format to ip, port
|
|
|
|
def unpackAddress(packed):
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
assert len(packed) == 6, "Invalid length ip4 packed address: %s" % len(packed)
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]
|
|
|
|
|
|
|
|
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
# onion, port to packed 12byte format
|
|
|
|
def packOnionAddress(onion, port):
|
|
|
|
onion = onion.replace(".onion", "")
|
|
|
|
return base64.b32decode(onion.upper()) + struct.pack("H", port)
|
|
|
|
|
|
|
|
|
|
|
|
# From 12byte format to ip, port
|
|
|
|
def unpackOnionAddress(packed):
|
|
|
|
return base64.b32encode(packed[0:-2]).lower() + ".onion", struct.unpack("H", packed[-2:])[0]
|
|
|
|
|
|
|
|
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
# Get dir from file
|
|
|
|
# Return: data/site/content.json -> data/site
|
|
|
|
def getDirname(path):
|
2016-04-06 13:48:13 +02:00
|
|
|
if "/" in path:
|
|
|
|
return path[:path.rfind("/")+1]
|
|
|
|
else:
|
|
|
|
return ""
|
Rev467, requirements.txt accept newer dependecies, Boost dbschema.json, Move getDirname getFilename to helper, Verify optional files, Includes not allowed in user files, Optional files rules, Peer hashfield functions, Test optional files signing, Test file info, Test verify file, Test helpers
2015-10-01 01:35:13 +02:00
|
|
|
|
|
|
|
# Get dir from file
|
|
|
|
# Return: data/site/content.json -> content.json
|
|
|
|
def getFilename(path):
|
2016-04-06 13:48:13 +02:00
|
|
|
return path[path.rfind("/")+1:]
|
2015-10-22 11:42:55 +02:00
|
|
|
|
|
|
|
|
|
|
|
# Convert hash to hashid for hashfield
|
|
|
|
def toHashId(hash):
|
|
|
|
return int(hash[0:4], 16)
|
|
|
|
|
|
|
|
|
|
|
|
# Merge dict values
|
|
|
|
def mergeDicts(dicts):
|
|
|
|
back = collections.defaultdict(set)
|
|
|
|
for d in dicts:
|
|
|
|
for key, val in d.iteritems():
|
|
|
|
back[key].update(val)
|
|
|
|
return dict(back)
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
|
|
|
|
|
|
|
# Request https url using gevent SSL error workaround
|
|
|
|
def httpRequest(url, as_file=False):
|
|
|
|
if url.startswith("http://"):
|
|
|
|
import urllib
|
|
|
|
response = urllib.urlopen(url)
|
|
|
|
else: # Hack to avoid Python gevent ssl errors
|
|
|
|
import socket
|
|
|
|
import httplib
|
|
|
|
import ssl
|
|
|
|
|
|
|
|
host, request = re.match("https://(.*?)(/.*?)$", url).groups()
|
|
|
|
|
|
|
|
conn = httplib.HTTPSConnection(host)
|
|
|
|
sock = socket.create_connection((conn.host, conn.port), conn.timeout, conn.source_address)
|
|
|
|
conn.sock = ssl.wrap_socket(sock, conn.key_file, conn.cert_file)
|
|
|
|
conn.request("GET", request)
|
|
|
|
response = conn.getresponse()
|
2016-03-16 22:07:11 +01:00
|
|
|
if response.status in [301, 302, 303, 307, 308]:
|
|
|
|
logging.info("Redirect to: %s" % response.getheader('Location'))
|
|
|
|
response = httpRequest(response.getheader('Location'))
|
Version 0.3.5, Rev830, Full Tor mode support with hidden services, Onion stats in Sidebar, GeoDB download fix using Tor, Gray out disabled sites in Stats page, Tor hidden service status in stat page, Benchmark sha256, Skyts tracker out expodie in, 2 new tracker using ZeroNet protocol, Keep SSL cert option between restarts, SSL Certificate pinning support for connections, Site lock support for connections, Certificate pinned connections using implicit SSL, Flood protection whitelist support, Foreign keys support for DB layer, Not support for SQL query helper, 0 length file get bugfix, Pex onion address support, Faster port testing, Faster uPnP port opening, Need connections more often on owned sites, Delay ZeroHello startup message if port check or Tor manager not ready yet, Use lockfiles to avoid double start, Save original socket on proxy monkey patching to get ability to connect localhost directly, Handle atomic write errors, Broken gevent https workaround helper, Rsa crypt functions, Plugin to Bootstrap using ZeroNet protocol
2016-01-05 00:20:52 +01:00
|
|
|
|
|
|
|
if as_file:
|
|
|
|
import cStringIO as StringIO
|
|
|
|
data = StringIO.StringIO()
|
|
|
|
while True:
|
|
|
|
buff = response.read(1024 * 16)
|
|
|
|
if not buff:
|
|
|
|
break
|
|
|
|
data.write(buff)
|
|
|
|
return data
|
|
|
|
else:
|
|
|
|
return response
|