Refactor a little
This commit is contained in:
parent
4ffe8b489e
commit
3fcc8dd998
|
@ -0,0 +1,12 @@
|
|||
USE mysql;
|
||||
|
||||
DROP user IF EXISTS wensleydale@localhost;
|
||||
|
||||
CREATE USER wensleydale@localhost
|
||||
IDENTIFIED BY '';
|
||||
|
||||
GRANT ALL PRIVILEGES ON cheese_shop.*
|
||||
TO 'wensleydale'@'localhost';
|
||||
|
||||
UPDATE user SET plugin='mysql_native_password' WHERE User='wensleydale';
|
||||
FLUSH PRIVILEGES;
|
|
@ -2,16 +2,16 @@ DROP DATABASE IF EXISTS cheese_shop;
|
|||
CREATE DATABASE cheese_shop;
|
||||
USE cheese_shop;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS releases (
|
||||
CREATE TABLE releases (
|
||||
id smallint AUTO_INCREMENT PRIMARY KEY,
|
||||
project varchar(32),
|
||||
version varchar(32));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS contacts (
|
||||
CREATE TABLE contacts (
|
||||
email varchar(255) PRIMARY KEY,
|
||||
name varchar(255));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS information (
|
||||
CREATE TABLE information (
|
||||
release_id smallint PRIMARY KEY,
|
||||
summary varchar(255),
|
||||
homepage varchar(2083),
|
||||
|
@ -19,24 +19,24 @@ CREATE TABLE IF NOT EXISTS information (
|
|||
FOREIGN KEY (release_id) REFERENCES releases(id),
|
||||
FOREIGN KEY (email) REFERENCES contacts(email));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS troves (
|
||||
CREATE TABLE troves (
|
||||
id smallint AUTO_INCREMENT PRIMARY KEY,
|
||||
classifier varchar(255));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS classifiers (
|
||||
CREATE TABLE classifiers (
|
||||
release_id smallint,
|
||||
trove_id smallint,
|
||||
PRIMARY KEY (release_id, trove_id),
|
||||
FOREIGN KEY (release_id) REFERENCES releases(id),
|
||||
FOREIGN KEY (trove_id) REFERENCES troves(id));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS keywords (
|
||||
CREATE TABLE keywords (
|
||||
release_id smallint,
|
||||
term varchar(32),
|
||||
PRIMARY KEY (release_id, term),
|
||||
FOREIGN KEY (release_id) REFERENCES releases(id));
|
||||
|
||||
CREATE TABLE IF NOT EXISTS distributions (
|
||||
CREATE TABLE distributions (
|
||||
release_id smallint,
|
||||
filename varchar(255),
|
||||
url varchar(255),
|
|
@ -1,48 +1,63 @@
|
|||
#!/usr/bin/env python3
|
||||
from typing import KeysView
|
||||
from typing import Any, Awaitable, KeysView
|
||||
|
||||
from asks.sessions import Session
|
||||
from mysql.connector import connect
|
||||
from trio import Nursery, open_nursery, run
|
||||
|
||||
PYPI = 'https://pypi.org'
|
||||
USER = 'wensleydale'
|
||||
DB = 'cheese_shop'
|
||||
INDEX = 'https://pypi.org'
|
||||
CONNECTIONS = 20
|
||||
|
||||
|
||||
async def culture(session: Session) -> KeysView[str]:
|
||||
"""Return the 100 most popular cheeses in cheese shop."""
|
||||
stats = await session.get(
|
||||
path='/stats', headers={'Accept': 'application/json'})
|
||||
return stats.json()['top_packages'].keys()
|
||||
class CheeseMaker:
|
||||
def __init__(self, nursery: Nursery, session: Session) -> None:
|
||||
self.nursery, self.session = nursery, session
|
||||
self.cursor = connect(user=USER, database=DB).cursor()
|
||||
|
||||
def start_soon(self, async_fn: Awaitable, *args: Any) -> None:
|
||||
"""Creates a child task, scheduling await async_fn(*args)."""
|
||||
self.nursery.start_soon(async_fn, *args)
|
||||
|
||||
async def drain(project_name: str, version: str, session: Session) -> None:
|
||||
"""Fetch metadata of the given distribution."""
|
||||
# XXX: insert the fetched metadata to a database
|
||||
await session.get(path=f'/pypi/{project_name}/{version}/json')
|
||||
print(project_name, version)
|
||||
async def json(self, path: str) -> Any:
|
||||
"""Return the JSON response to the given GET request."""
|
||||
response = await self.session.get(
|
||||
path=path, headers={'Accept': 'application/json'})
|
||||
return response.json()
|
||||
|
||||
async def culture(self) -> KeysView[str]:
|
||||
"""Return the 100 most popular cheeses in cheese shop."""
|
||||
stats = await self.json('/stats')
|
||||
return stats['top_packages'].keys()
|
||||
|
||||
async def coagulate(project_name: str, nursery: Nursery,
|
||||
session: Session) -> None:
|
||||
"""Fetch project's available versions and the metadata of each."""
|
||||
response = await session.get(path=f'/pypi/{project_name}/json')
|
||||
for version in response.json()['releases'].keys():
|
||||
# Recklessly filter out prereleases
|
||||
for n in version.split('.'):
|
||||
try:
|
||||
int(n)
|
||||
except ValueError:
|
||||
break
|
||||
else:
|
||||
nursery.start_soon(drain, project_name, version, session)
|
||||
async def drain(self, project_name: str, version: str) -> None:
|
||||
"""Fetch metadata of the given distribution."""
|
||||
# XXX: insert the fetched metadata to a database
|
||||
await self.json(f'/pypi/{project_name}/{version}/json')
|
||||
print(project_name, version)
|
||||
|
||||
async def coagulate(self, project_name: str) -> None:
|
||||
"""Fetch project's available versions and metadata."""
|
||||
response = await self.json(f'/pypi/{project_name}/json')
|
||||
for version in response['releases'].keys():
|
||||
# Recklessly filter out prereleases
|
||||
for n in version.split('.'):
|
||||
try:
|
||||
int(n)
|
||||
except ValueError:
|
||||
break
|
||||
else:
|
||||
self.start_soon(self.drain, project_name, version)
|
||||
|
||||
|
||||
async def main(session: Session):
|
||||
"""Make cheeses."""
|
||||
async with open_nursery() as nursery:
|
||||
for project_name in await culture(session):
|
||||
nursery.start_soon(coagulate, project_name, nursery, session)
|
||||
maker = CheeseMaker(nursery, session)
|
||||
for project_name in await maker.culture():
|
||||
maker.start_soon(maker.coagulate, project_name)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main, Session(PYPI, connections=CONNECTIONS))
|
||||
run(main, Session(INDEX, connections=CONNECTIONS))
|
||||
|
|
|
@ -1,2 +1,3 @@
|
|||
asks
|
||||
mysql.connector
|
||||
trio
|
||||
|
|
Loading…
Reference in New Issue