ZeroNet/src/util/helper.py

import os
import stat
import socket
import struct
import re
import collections
import time
import logging
import base64
import gevent

if "inet_pton" not in dir(socket):
    import win_inet_pton

from Config import config


def atomicWrite(dest, content, mode="w"):
    try:
        with open(dest + "-tmpnew", mode) as f:
            f.write(content)
            f.flush()
            os.fsync(f.fileno())
        if os.path.isfile(dest + "-tmpold"):  # Previous incomplete write
            os.rename(dest + "-tmpold", dest + "-tmpold-%s" % time.time())
        if os.path.isfile(dest):  # Rename old file to -tmpold
            os.rename(dest, dest + "-tmpold")
        os.rename(dest + "-tmpnew", dest)
        os.unlink(dest + "-tmpold")  # Remove old file
        return True
    except Exception, err:
        from Debug import Debug
        logging.error(
            "File %s write failed: %s, reverting..." %
            (dest, Debug.formatException(err))
        )
        if os.path.isfile(dest + "-tmpold") and not os.path.isfile(dest):
            os.rename(dest + "-tmpold", dest)
        return False


def openLocked(path, mode="w"):
    if os.name == "posix":
        import fcntl
        f = open(path, mode)
        fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
    elif os.name == "nt":
        import msvcrt
        f = open(path, mode)
        msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, -1)
    else:
        f = open(path, mode)
    return f


def getFreeSpace():
    free_space = -1
    if "statvfs" in dir(os):  # Unix
        statvfs = os.statvfs(config.data_dir.encode("utf8"))
        free_space = statvfs.f_frsize * statvfs.f_bavail
    else:  # Windows
        try:
            import ctypes
            free_space_pointer = ctypes.c_ulonglong(0)
            ctypes.windll.kernel32.GetDiskFreeSpaceExW(
                ctypes.c_wchar_p(config.data_dir), None, None, ctypes.pointer(free_space_pointer)
            )
            free_space = free_space_pointer.value
        except Exception, err:
            logging.error("GetFreeSpace error: %s" % err)
    return free_space


def shellquote(*args):
    if len(args) == 1:
        return '"%s"' % args[0].replace('"', "")
    else:
        return tuple(['"%s"' % arg.replace('"', "") for arg in args])


def packPeers(peers):
    packed_peers = {"ipv4": [], "ipv6": [], "onion": []}
    for peer in peers:
        try:
            ip_type = getIpType(peer.ip)
            packed_peers[ip_type].append(peer.packMyAddress())
        except Exception:
            logging.error("Error packing peer address: %s" % peer)
    return packed_peers


# ip, port to packed 6byte or 18byte format
def packAddress(ip, port):
    if ":" in ip:
        return socket.inet_pton(socket.AF_INET6, ip) + struct.pack("H", port)
    else:
        return socket.inet_aton(ip) + struct.pack("H", port)


# From 6byte or 18byte format to ip, port
def unpackAddress(packed):
    if len(packed) == 18:
        return socket.inet_ntop(socket.AF_INET6, packed[0:16]), struct.unpack_from("H", packed, 16)[0]
    else:
        assert len(packed) == 6, "Invalid length ip4 packed address: %s" % len(packed)
        return socket.inet_ntoa(packed[0:4]), struct.unpack_from("H", packed, 4)[0]


# onion, port to packed 12byte format
def packOnionAddress(onion, port):
    onion = onion.replace(".onion", "")
    return base64.b32decode(onion.upper()) + struct.pack("H", port)


# From 12byte format to ip, port
def unpackOnionAddress(packed):
    return base64.b32encode(packed[0:-2]).lower() + ".onion", struct.unpack("H", packed[-2:])[0]


# Get dir from file
# Return: data/site/content.json -> data/site/
def getDirname(path):
    if "/" in path:
        return path[:path.rfind("/") + 1].lstrip("/")
    else:
        return ""


# Get dir from file
# Return: data/site/content.json -> content.json
def getFilename(path):
    return path[path.rfind("/") + 1:]


def getFilesize(path):
    try:
        s = os.stat(path)
    except:
        return None
    if stat.S_ISREG(s.st_mode):  # Test if it's file
        return s.st_size
    else:
        return None

# Convert hash to hashid for hashfield
def toHashId(hash):
    return int(hash[0:4], 16)


# Merge dict values
def mergeDicts(dicts):
    back = collections.defaultdict(set)
    for d in dicts:
        for key, val in d.iteritems():
            back[key].update(val)
    return dict(back)


# Request https url using gevent SSL error workaround
def httpRequest(url, as_file=False):
    if url.startswith("http://"):
        import urllib
        response = urllib.urlopen(url)
    else:  # Hack to avoid Python gevent ssl errors
        import socket
        import httplib
        import ssl

        host, request = re.match("https://(.*?)(/.*?)$", url).groups()

        conn = httplib.HTTPSConnection(host)
        sock = socket.create_connection((conn.host, conn.port), conn.timeout, conn.source_address)
        conn.sock = ssl.wrap_socket(sock, conn.key_file, conn.cert_file)
        conn.request("GET", request)
        response = conn.getresponse()
        if response.status in [301, 302, 303, 307, 308]:
            logging.info("Redirect to: %s" % response.getheader('Location'))
            response = httpRequest(response.getheader('Location'))

    if as_file:
        import cStringIO as StringIO
        data = StringIO.StringIO()
        while True:
            buff = response.read(1024 * 16)
            if not buff:
                break
            data.write(buff)
        return data
    else:
        return response


def timerCaller(secs, func, *args, **kwargs):
    gevent.spawn_later(secs, timerCaller, secs, func, *args, **kwargs)
    func(*args, **kwargs)


def timer(secs, func, *args, **kwargs):
    gevent.spawn_later(secs, timerCaller, secs, func, *args, **kwargs)


def create_connection(address, timeout=None, source_address=None):
    if address in config.ip_local:
        sock = socket.create_connection_original(address, timeout, source_address)
    else:
        sock = socket.create_connection_original(address, timeout, socket.bind_addr)
    return sock

def socketBindMonkeyPatch(bind_ip, bind_port):
    import socket
    logging.info("Monkey patching socket to bind to: %s:%s" % (bind_ip, bind_port))
    socket.bind_addr = (bind_ip, int(bind_port))
    socket.create_connection_original = socket.create_connection
    socket.create_connection = create_connection


def limitedGzipFile(*args, **kwargs):
    import gzip
    class LimitedGzipFile(gzip.GzipFile):
        def read(self, size=-1):
            return super(LimitedGzipFile, self).read(1024*1024*25)
    return LimitedGzipFile(*args, **kwargs)

def avg(items):
    if len(items) > 0:
        return sum(items) / len(items)
    else:
        return 0

def isIp(ip):
    if ":" in ip:  # IPv6
        try:
            socket.inet_pton(socket.AF_INET6, ip)
            return True
        except:
            return False

    else:  # IPv4
        try:
            socket.inet_aton(ip)
            return True
        except:
            return False

local_ip_pattern = re.compile(r"^(127\.)|(192\.168\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(::1$)|([fF][cCdD])")
def isPrivateIp(ip):
    return local_ip_pattern.match(ip)