lazip/lazip.py

# Lazy ZIP over HTTP
# Copyright (C) 2020  Nguyễn Gia Phong
#
# This file is part of lazip.
#
# lazip is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# lazip is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with lazip.  If not, see <https://www.gnu.org/licenses/>.

"""Lazy ZIP over HTTP"""

__version__ = '0.0.5'
__all__ = ['Filazy', 'Lazip']

from abc import abstractmethod
from bisect import bisect_left, bisect_right
from contextlib import contextmanager
from io import UnsupportedOperation
from tempfile import NamedTemporaryFile
from typing import IO, Dict, Iterator, List, Optional, Tuple
from zipfile import BadZipFile, ZipFile

from requests import Session
from requests.models import CONTENT_CHUNK_SIZE, Response


class ReadOnlyBinaryIOWrapper(IO[bytes]):
    """Wrapper for a read-only binary I/O."""

    file: IO[bytes]
    length: int

    @property
    def mode(self) -> str:
        """Opening mode, which is always rb."""
        return 'rb'

    @property
    def name(self) -> str:
        """Path to the underlying file."""
        return self.file.name

    def close(self) -> None:
        """Close the file."""
        self.file.close()

    @property
    def closed(self) -> bool:
        """Whether the file is closed."""
        return self.file.closed

    def fileno(self) -> int:
        """Return the underlying file descriptor (an integer)."""
        return self.file.fileno()

    def flush(self) -> None:
        """Do nothing."""
        self.file.flush()

    def isatty(self) -> bool:
        """Return False."""
        return self.file.isatty()

    def read(self, size: int = -1) -> bytes:
        """Read up to size bytes from the object and return them.

        As a convenience, if size is unspecified or -1,
        all bytes until EOF are returned.  Fewer than
        size bytes may be returned if EOF is reached.
        """
        start = self.tell()
        stop = start + size if 0 <= size <= self.length-start else self.length
        self.ensure(start, stop-1)
        return self.file.read(size)

    def readable(self) -> bool:
        """Return True."""
        return self.file.readable()

    def readline(self, limit):
        raise UnsupportedOperation

    def readlines(self, hint):
        raise UnsupportedOperation

    def seek(self, offset: int, whence: int = 0) -> int:
        """Change stream position and return the new absolute position.

        Seek to offset relative position indicated by whence:
        * 0: Start of stream (the default).  pos should be >= 0;
        * 1: Current position - pos may be negative;
        * 2: End of stream - pos usually negative.
        """
        return self.file.seek(offset, whence)

    def seekable(self) -> bool:
        """Return whether random access is supported, which is True."""
        return self.file.seekable()

    def tell(self) -> int:
        """Return the current possition."""
        return self.file.tell()

    def truncate(self, size: Optional[int] = None) -> int:
        """Resize the stream to the given size in bytes.

        If size is unspecified resize to the current position.
        The current stream position isn't changed.

        Return the new file size.
        """
        return self.file.truncate(size)

    def writable(self) -> bool:
        """Return False."""
        return False

    def write(self, s):
        raise UnsupportedOperation

    def writelines(self, lines):
        raise UnsupportedOperation

    def __next__(self):
        raise UnsupportedOperation

    def __iter__(self):
        raise UnsupportedOperation

    def __enter__(self) -> 'ReadOnlyBinaryIOWrapper':
        self.file.__enter__()
        return self

    def __exit__(self, *exc) -> Optional[bool]:
        return self.file.__exit__(*exc)

    @abstractmethod
    def ensure(self, start: int, end: int) -> None:
        """Ensure the data from start to end inclusively.

        This method must return to the original position
        if seek is called.
        """


class Filazy(ReadOnlyBinaryIOWrapper):
    """Read-only file-like object mapped to a file over HTTP.

    This uses HTTP range requests to lazily fetch the file's content.
    At the end of initialization, __post_init__ will be called.

    Parameters:
        session (Session): Requests session
        url (str): HTTP URL to the file
        chunk_size (int): Download chunk size

    Attributes:
        session (Session): Requests session
        url (str): HTTP URL to the file
        chunk_size (int): Download chunk size
        left (List[int]): Left endpoints of downloaded intervals
        right (List[int]): Right endpoints of downloaded intervals
        accept_ranges (bool): Whether range requests are supported
    """

    def __init__(self, url: str, session: Session,
                 chunk_size: int = CONTENT_CHUNK_SIZE) -> None:
        response = session.head(url)
        response.raise_for_status()
        assert response.status_code == 200
        headers = response.headers
        self.session, self.url, self.chunk_size = session, url, chunk_size
        self.length = int(headers['Content-Length'])
        self.file = NamedTemporaryFile()
        self.truncate(self.length)
        self.left: List[int] = []
        self.right: List[int] = []
        self.accept_ranges = 'bytes' in headers.get('Accept-Ranges', 'none')
        with self.stay(): self.__post_init__()

    def __post_init__(self) -> None:
        pass

    @contextmanager
    def stay(self) -> Iterator[None]:
        """Return a context manager that keeps the stream position.

        At the end of the block, seek back to original position.
        """
        pos = self.tell()
        try:
            yield
        finally:
            self.seek(pos)

    def stream_response(self, start: int, end: int,
                        base_headers: Dict[str, str] = {}) -> Response:
        """Return HTTP response to a range request from start to end."""
        headers = {'Range': f'bytes={start}-{end}', **base_headers}
        return self.session.get(self.url, headers=headers, stream=True)

    def merge(self, start: int, end: int,
              left: int, right: int) -> Iterator[Tuple[int, int]]:
        """Return an iterator of intervals to be fetched.

        Parameters:
            start (int): Start of needed interval
            end (int): End of needed interval
            left (int): Index of first overlapping downloaded data
            right (int): Index after last overlapping downloaded data
        """
        lslice, rslice = self.left[left:right], self.right[left:right]
        i = start = min([start, *lslice[:1]])
        end = max([end, *rslice[-1:]])
        for j, k in zip(lslice, rslice):
            if j > i: yield i, j-1
            i = k + 1
        if i <= end: yield i, end
        self.left[left:right], self.right[left:right] = [start], [end]

    def ensure(self, start: int, end: int) -> None:
        """Download bytes from start to end inclusively."""
        offset = self.chunk_size - 1
        start = max(0, min(start, end-offset))
        end = min(self.length-1, max(end, start+offset))
        with self.stay():
            i, j = bisect_left(self.right, start), bisect_right(self.left, end)
            for start, end in self.merge(start, end, i, j):
                response = self.stream_response(start, end)
                response.raise_for_status()
                self.seek(start)
                for chunk in response.raw.stream(self.chunk_size,
                                                 decode_content=False):
                    self.file.write(chunk)


class Lazip(Filazy):
    """Read-only file-like object mapped to a ZIP file over HTTP.

    This uses HTTP range requests to lazily fetch the file's content,
    which is supposed to be fed to ZipFile.
    """

    def __post_init__(self) -> None:
        """Check and download until the file is a valid ZIP."""
        end = self.length - 1
        if not self.accept_ranges:
            self.ensure(0, end)
            self.left, self.right = [0], [end]
            return
        for start in reversed(range(0, end, self.chunk_size)):
            self.ensure(start, end)
            try:
                ZipFile(self)
            except BadZipFile:
                pass
            else:
                break