Draft a way to fetch metadata from PyPI
This commit is contained in:
parent
78110b68c5
commit
f40ad912ff
|
@ -0,0 +1,5 @@
|
||||||
|
branches:
|
||||||
|
only: master
|
||||||
|
language: python
|
||||||
|
install: pip install tox
|
||||||
|
script: tox
|
|
@ -0,0 +1,48 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from typing import KeysView
|
||||||
|
|
||||||
|
from asks.sessions import Session
|
||||||
|
from trio import Nursery, open_nursery, run
|
||||||
|
|
||||||
|
PYPI = 'https://pypi.org'
|
||||||
|
CONNECTIONS = 20
|
||||||
|
|
||||||
|
|
||||||
|
async def culture(session: Session) -> KeysView[str]:
|
||||||
|
"""Return the 100 most popular cheeses in cheese shop."""
|
||||||
|
stats = await session.get(
|
||||||
|
path='/stats', headers={'Accept': 'application/json'})
|
||||||
|
return stats.json()['top_packages'].keys()
|
||||||
|
|
||||||
|
|
||||||
|
async def drain(project_name: str, version: str, session: Session) -> None:
|
||||||
|
"""Fetch metadata of the given distribution."""
|
||||||
|
# XXX: insert the fetched metadata to a database
|
||||||
|
await session.get(path=f'/pypi/{project_name}/{version}/json')
|
||||||
|
print(project_name, version)
|
||||||
|
|
||||||
|
|
||||||
|
async def coagulate(project_name: str, nursery: Nursery,
|
||||||
|
session: Session) -> None:
|
||||||
|
"""Fetch project's available versions and the metadata of each."""
|
||||||
|
response = await session.get(path=f'/pypi/{project_name}/json')
|
||||||
|
for version in response.json()['releases'].keys():
|
||||||
|
# Recklessly filter out prereleases
|
||||||
|
for n in version.split('.'):
|
||||||
|
try:
|
||||||
|
int(n)
|
||||||
|
except ValueError:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
nursery.start_soon(drain, project_name, version, session)
|
||||||
|
|
||||||
|
|
||||||
|
async def main(session: Session):
|
||||||
|
"""Make cheeses."""
|
||||||
|
async with open_nursery() as nursery:
|
||||||
|
for project_name in await culture(session):
|
||||||
|
nursery.start_soon(coagulate, project_name, nursery, session)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
run(main, Session(PYPI, connections=CONNECTIONS))
|
|
@ -0,0 +1,2 @@
|
||||||
|
asks
|
||||||
|
trio
|
|
@ -0,0 +1,19 @@
|
||||||
|
[tox]
|
||||||
|
envlist = py
|
||||||
|
minversion = 3.3
|
||||||
|
isolated_build = True
|
||||||
|
|
||||||
|
[testenv]
|
||||||
|
skip_install = True
|
||||||
|
deps =
|
||||||
|
flake8-builtins
|
||||||
|
isort[requirements]
|
||||||
|
commands =
|
||||||
|
flake8
|
||||||
|
isort -c --diff
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
hang-closing = True
|
||||||
|
ignore = W503, E125, E225, E226, E227, E701, E704
|
||||||
|
; See https://github.com/PyCQA/pycodestyle/issues/906
|
||||||
|
;max-doc-length = 72
|
Loading…
Reference in New Issue