Draft a way to fetch metadata from PyPI
This commit is contained in:
parent
78110b68c5
commit
f40ad912ff
|
@ -0,0 +1,5 @@
|
|||
branches:
|
||||
only: master
|
||||
language: python
|
||||
install: pip install tox
|
||||
script: tox
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env python3
|
||||
from typing import KeysView
|
||||
|
||||
from asks.sessions import Session
|
||||
from trio import Nursery, open_nursery, run
|
||||
|
||||
PYPI = 'https://pypi.org'
|
||||
CONNECTIONS = 20
|
||||
|
||||
|
||||
async def culture(session: Session) -> KeysView[str]:
|
||||
"""Return the 100 most popular cheeses in cheese shop."""
|
||||
stats = await session.get(
|
||||
path='/stats', headers={'Accept': 'application/json'})
|
||||
return stats.json()['top_packages'].keys()
|
||||
|
||||
|
||||
async def drain(project_name: str, version: str, session: Session) -> None:
|
||||
"""Fetch metadata of the given distribution."""
|
||||
# XXX: insert the fetched metadata to a database
|
||||
await session.get(path=f'/pypi/{project_name}/{version}/json')
|
||||
print(project_name, version)
|
||||
|
||||
|
||||
async def coagulate(project_name: str, nursery: Nursery,
|
||||
session: Session) -> None:
|
||||
"""Fetch project's available versions and the metadata of each."""
|
||||
response = await session.get(path=f'/pypi/{project_name}/json')
|
||||
for version in response.json()['releases'].keys():
|
||||
# Recklessly filter out prereleases
|
||||
for n in version.split('.'):
|
||||
try:
|
||||
int(n)
|
||||
except ValueError:
|
||||
break
|
||||
else:
|
||||
nursery.start_soon(drain, project_name, version, session)
|
||||
|
||||
|
||||
async def main(session: Session):
|
||||
"""Make cheeses."""
|
||||
async with open_nursery() as nursery:
|
||||
for project_name in await culture(session):
|
||||
nursery.start_soon(coagulate, project_name, nursery, session)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main, Session(PYPI, connections=CONNECTIONS))
|
|
@ -0,0 +1,2 @@
|
|||
asks
|
||||
trio
|
|
@ -0,0 +1,19 @@
|
|||
[tox]
|
||||
envlist = py
|
||||
minversion = 3.3
|
||||
isolated_build = True
|
||||
|
||||
[testenv]
|
||||
skip_install = True
|
||||
deps =
|
||||
flake8-builtins
|
||||
isort[requirements]
|
||||
commands =
|
||||
flake8
|
||||
isort -c --diff
|
||||
|
||||
[flake8]
|
||||
hang-closing = True
|
||||
ignore = W503, E125, E225, E226, E227, E701, E704
|
||||
; See https://github.com/PyCQA/pycodestyle/issues/906
|
||||
;max-doc-length = 72
|
Loading…
Reference in New Issue