diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2baaa4f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,5 @@ +branches: + only: master +language: python +install: pip install tox +script: tox diff --git a/tools/make-cheeses.py b/tools/make-cheeses.py new file mode 100755 index 0000000..b7327af --- /dev/null +++ b/tools/make-cheeses.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +from typing import KeysView + +from asks.sessions import Session +from trio import Nursery, open_nursery, run + +PYPI = 'https://pypi.org' +CONNECTIONS = 20 + + +async def culture(session: Session) -> KeysView[str]: + """Return the 100 most popular cheeses in cheese shop.""" + stats = await session.get( + path='/stats', headers={'Accept': 'application/json'}) + return stats.json()['top_packages'].keys() + + +async def drain(project_name: str, version: str, session: Session) -> None: + """Fetch metadata of the given distribution.""" + # XXX: insert the fetched metadata to a database + await session.get(path=f'/pypi/{project_name}/{version}/json') + print(project_name, version) + + +async def coagulate(project_name: str, nursery: Nursery, + session: Session) -> None: + """Fetch project's available versions and the metadata of each.""" + response = await session.get(path=f'/pypi/{project_name}/json') + for version in response.json()['releases'].keys(): + # Recklessly filter out prereleases + for n in version.split('.'): + try: + int(n) + except ValueError: + break + else: + nursery.start_soon(drain, project_name, version, session) + + +async def main(session: Session): + """Make cheeses.""" + async with open_nursery() as nursery: + for project_name in await culture(session): + nursery.start_soon(coagulate, project_name, nursery, session) + + +if __name__ == '__main__': + run(main, Session(PYPI, connections=CONNECTIONS)) diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 0000000..559c85b --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,2 @@ +asks +trio diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..79488a6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,19 @@ +[tox] +envlist = py +minversion = 3.3 +isolated_build = True + +[testenv] +skip_install = True +deps = + flake8-builtins + isort[requirements] +commands = + flake8 + isort -c --diff + +[flake8] +hang-closing = True +ignore = W503, E125, E225, E226, E227, E701, E704 +; See https://github.com/PyCQA/pycodestyle/issues/906 +;max-doc-length = 72