Browse Source

Draft a way to fetch metadata from PyPI

master
Nguyễn Gia Phong 2 years ago
parent
commit
f40ad912ff
  1. 5
      .travis.yml
  2. 48
      tools/make-cheeses.py
  3. 2
      tools/requirements.txt
  4. 19
      tox.ini

5
.travis.yml

@ -0,0 +1,5 @@
branches:
only: master
language: python
install: pip install tox
script: tox

48
tools/make-cheeses.py

@ -0,0 +1,48 @@
#!/usr/bin/env python3
from typing import KeysView
from asks.sessions import Session
from trio import Nursery, open_nursery, run
PYPI = 'https://pypi.org'
CONNECTIONS = 20
async def culture(session: Session) -> KeysView[str]:
"""Return the 100 most popular cheeses in cheese shop."""
stats = await session.get(
path='/stats', headers={'Accept': 'application/json'})
return stats.json()['top_packages'].keys()
async def drain(project_name: str, version: str, session: Session) -> None:
"""Fetch metadata of the given distribution."""
# XXX: insert the fetched metadata to a database
await session.get(path=f'/pypi/{project_name}/{version}/json')
print(project_name, version)
async def coagulate(project_name: str, nursery: Nursery,
session: Session) -> None:
"""Fetch project's available versions and the metadata of each."""
response = await session.get(path=f'/pypi/{project_name}/json')
for version in response.json()['releases'].keys():
# Recklessly filter out prereleases
for n in version.split('.'):
try:
int(n)
except ValueError:
break
else:
nursery.start_soon(drain, project_name, version, session)
async def main(session: Session):
"""Make cheeses."""
async with open_nursery() as nursery:
for project_name in await culture(session):
nursery.start_soon(coagulate, project_name, nursery, session)
if __name__ == '__main__':
run(main, Session(PYPI, connections=CONNECTIONS))

2
tools/requirements.txt

@ -0,0 +1,2 @@
asks
trio

19
tox.ini

@ -0,0 +1,19 @@
[tox]
envlist = py
minversion = 3.3
isolated_build = True
[testenv]
skip_install = True
deps =
flake8-builtins
isort[requirements]
commands =
flake8
isort -c --diff
[flake8]
hang-closing = True
ignore = W503, E125, E225, E226, E227, E701, E704
; See https://github.com/PyCQA/pycodestyle/issues/906
;max-doc-length = 72
Loading…
Cancel
Save