Refactor a little
This commit is contained in:
parent
4ffe8b489e
commit
3fcc8dd998
|
@ -0,0 +1,12 @@
|
||||||
|
USE mysql;
|
||||||
|
|
||||||
|
DROP user IF EXISTS wensleydale@localhost;
|
||||||
|
|
||||||
|
CREATE USER wensleydale@localhost
|
||||||
|
IDENTIFIED BY '';
|
||||||
|
|
||||||
|
GRANT ALL PRIVILEGES ON cheese_shop.*
|
||||||
|
TO 'wensleydale'@'localhost';
|
||||||
|
|
||||||
|
UPDATE user SET plugin='mysql_native_password' WHERE User='wensleydale';
|
||||||
|
FLUSH PRIVILEGES;
|
|
@ -2,16 +2,16 @@ DROP DATABASE IF EXISTS cheese_shop;
|
||||||
CREATE DATABASE cheese_shop;
|
CREATE DATABASE cheese_shop;
|
||||||
USE cheese_shop;
|
USE cheese_shop;
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS releases (
|
CREATE TABLE releases (
|
||||||
id smallint AUTO_INCREMENT PRIMARY KEY,
|
id smallint AUTO_INCREMENT PRIMARY KEY,
|
||||||
project varchar(32),
|
project varchar(32),
|
||||||
version varchar(32));
|
version varchar(32));
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS contacts (
|
CREATE TABLE contacts (
|
||||||
email varchar(255) PRIMARY KEY,
|
email varchar(255) PRIMARY KEY,
|
||||||
name varchar(255));
|
name varchar(255));
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS information (
|
CREATE TABLE information (
|
||||||
release_id smallint PRIMARY KEY,
|
release_id smallint PRIMARY KEY,
|
||||||
summary varchar(255),
|
summary varchar(255),
|
||||||
homepage varchar(2083),
|
homepage varchar(2083),
|
||||||
|
@ -19,24 +19,24 @@ CREATE TABLE IF NOT EXISTS information (
|
||||||
FOREIGN KEY (release_id) REFERENCES releases(id),
|
FOREIGN KEY (release_id) REFERENCES releases(id),
|
||||||
FOREIGN KEY (email) REFERENCES contacts(email));
|
FOREIGN KEY (email) REFERENCES contacts(email));
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS troves (
|
CREATE TABLE troves (
|
||||||
id smallint AUTO_INCREMENT PRIMARY KEY,
|
id smallint AUTO_INCREMENT PRIMARY KEY,
|
||||||
classifier varchar(255));
|
classifier varchar(255));
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS classifiers (
|
CREATE TABLE classifiers (
|
||||||
release_id smallint,
|
release_id smallint,
|
||||||
trove_id smallint,
|
trove_id smallint,
|
||||||
PRIMARY KEY (release_id, trove_id),
|
PRIMARY KEY (release_id, trove_id),
|
||||||
FOREIGN KEY (release_id) REFERENCES releases(id),
|
FOREIGN KEY (release_id) REFERENCES releases(id),
|
||||||
FOREIGN KEY (trove_id) REFERENCES troves(id));
|
FOREIGN KEY (trove_id) REFERENCES troves(id));
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS keywords (
|
CREATE TABLE keywords (
|
||||||
release_id smallint,
|
release_id smallint,
|
||||||
term varchar(32),
|
term varchar(32),
|
||||||
PRIMARY KEY (release_id, term),
|
PRIMARY KEY (release_id, term),
|
||||||
FOREIGN KEY (release_id) REFERENCES releases(id));
|
FOREIGN KEY (release_id) REFERENCES releases(id));
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS distributions (
|
CREATE TABLE distributions (
|
||||||
release_id smallint,
|
release_id smallint,
|
||||||
filename varchar(255),
|
filename varchar(255),
|
||||||
url varchar(255),
|
url varchar(255),
|
|
@ -1,48 +1,63 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
from typing import KeysView
|
from typing import Any, Awaitable, KeysView
|
||||||
|
|
||||||
from asks.sessions import Session
|
from asks.sessions import Session
|
||||||
|
from mysql.connector import connect
|
||||||
from trio import Nursery, open_nursery, run
|
from trio import Nursery, open_nursery, run
|
||||||
|
|
||||||
PYPI = 'https://pypi.org'
|
USER = 'wensleydale'
|
||||||
|
DB = 'cheese_shop'
|
||||||
|
INDEX = 'https://pypi.org'
|
||||||
CONNECTIONS = 20
|
CONNECTIONS = 20
|
||||||
|
|
||||||
|
|
||||||
async def culture(session: Session) -> KeysView[str]:
|
class CheeseMaker:
|
||||||
"""Return the 100 most popular cheeses in cheese shop."""
|
def __init__(self, nursery: Nursery, session: Session) -> None:
|
||||||
stats = await session.get(
|
self.nursery, self.session = nursery, session
|
||||||
path='/stats', headers={'Accept': 'application/json'})
|
self.cursor = connect(user=USER, database=DB).cursor()
|
||||||
return stats.json()['top_packages'].keys()
|
|
||||||
|
|
||||||
|
def start_soon(self, async_fn: Awaitable, *args: Any) -> None:
|
||||||
|
"""Creates a child task, scheduling await async_fn(*args)."""
|
||||||
|
self.nursery.start_soon(async_fn, *args)
|
||||||
|
|
||||||
async def drain(project_name: str, version: str, session: Session) -> None:
|
async def json(self, path: str) -> Any:
|
||||||
"""Fetch metadata of the given distribution."""
|
"""Return the JSON response to the given GET request."""
|
||||||
# XXX: insert the fetched metadata to a database
|
response = await self.session.get(
|
||||||
await session.get(path=f'/pypi/{project_name}/{version}/json')
|
path=path, headers={'Accept': 'application/json'})
|
||||||
print(project_name, version)
|
return response.json()
|
||||||
|
|
||||||
|
async def culture(self) -> KeysView[str]:
|
||||||
|
"""Return the 100 most popular cheeses in cheese shop."""
|
||||||
|
stats = await self.json('/stats')
|
||||||
|
return stats['top_packages'].keys()
|
||||||
|
|
||||||
async def coagulate(project_name: str, nursery: Nursery,
|
async def drain(self, project_name: str, version: str) -> None:
|
||||||
session: Session) -> None:
|
"""Fetch metadata of the given distribution."""
|
||||||
"""Fetch project's available versions and the metadata of each."""
|
# XXX: insert the fetched metadata to a database
|
||||||
response = await session.get(path=f'/pypi/{project_name}/json')
|
await self.json(f'/pypi/{project_name}/{version}/json')
|
||||||
for version in response.json()['releases'].keys():
|
print(project_name, version)
|
||||||
# Recklessly filter out prereleases
|
|
||||||
for n in version.split('.'):
|
async def coagulate(self, project_name: str) -> None:
|
||||||
try:
|
"""Fetch project's available versions and metadata."""
|
||||||
int(n)
|
response = await self.json(f'/pypi/{project_name}/json')
|
||||||
except ValueError:
|
for version in response['releases'].keys():
|
||||||
break
|
# Recklessly filter out prereleases
|
||||||
else:
|
for n in version.split('.'):
|
||||||
nursery.start_soon(drain, project_name, version, session)
|
try:
|
||||||
|
int(n)
|
||||||
|
except ValueError:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
self.start_soon(self.drain, project_name, version)
|
||||||
|
|
||||||
|
|
||||||
async def main(session: Session):
|
async def main(session: Session):
|
||||||
"""Make cheeses."""
|
"""Make cheeses."""
|
||||||
async with open_nursery() as nursery:
|
async with open_nursery() as nursery:
|
||||||
for project_name in await culture(session):
|
maker = CheeseMaker(nursery, session)
|
||||||
nursery.start_soon(coagulate, project_name, nursery, session)
|
for project_name in await maker.culture():
|
||||||
|
maker.start_soon(maker.coagulate, project_name)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
run(main, Session(PYPI, connections=CONNECTIONS))
|
run(main, Session(INDEX, connections=CONNECTIONS))
|
||||||
|
|
|
@ -1,2 +1,3 @@
|
||||||
asks
|
asks
|
||||||
|
mysql.connector
|
||||||
trio
|
trio
|
||||||
|
|
Loading…
Reference in New Issue