Browse Source

Refactor a little

master
Nguyễn Gia Phong 2 years ago
parent
commit
3fcc8dd998
  1. 12
      sql/ctl.sql
  2. 14
      sql/def.sql
  3. 79
      tools/make-cheeses.py
  4. 1
      tools/requirements.txt
  5. 2
      tox.ini

12
sql/ctl.sql

@ -0,0 +1,12 @@
USE mysql;
DROP user IF EXISTS wensleydale@localhost;
CREATE USER wensleydale@localhost
IDENTIFIED BY '';
GRANT ALL PRIVILEGES ON cheese_shop.*
TO 'wensleydale'@'localhost';
UPDATE user SET plugin='mysql_native_password' WHERE User='wensleydale';
FLUSH PRIVILEGES;

14
tools/def.sql → sql/def.sql

@ -2,16 +2,16 @@ DROP DATABASE IF EXISTS cheese_shop;
CREATE DATABASE cheese_shop;
USE cheese_shop;
CREATE TABLE IF NOT EXISTS releases (
CREATE TABLE releases (
id smallint AUTO_INCREMENT PRIMARY KEY,
project varchar(32),
version varchar(32));
CREATE TABLE IF NOT EXISTS contacts (
CREATE TABLE contacts (
email varchar(255) PRIMARY KEY,
name varchar(255));
CREATE TABLE IF NOT EXISTS information (
CREATE TABLE information (
release_id smallint PRIMARY KEY,
summary varchar(255),
homepage varchar(2083),
@ -19,24 +19,24 @@ CREATE TABLE IF NOT EXISTS information (
FOREIGN KEY (release_id) REFERENCES releases(id),
FOREIGN KEY (email) REFERENCES contacts(email));
CREATE TABLE IF NOT EXISTS troves (
CREATE TABLE troves (
id smallint AUTO_INCREMENT PRIMARY KEY,
classifier varchar(255));
CREATE TABLE IF NOT EXISTS classifiers (
CREATE TABLE classifiers (
release_id smallint,
trove_id smallint,
PRIMARY KEY (release_id, trove_id),
FOREIGN KEY (release_id) REFERENCES releases(id),
FOREIGN KEY (trove_id) REFERENCES troves(id));
CREATE TABLE IF NOT EXISTS keywords (
CREATE TABLE keywords (
release_id smallint,
term varchar(32),
PRIMARY KEY (release_id, term),
FOREIGN KEY (release_id) REFERENCES releases(id));
CREATE TABLE IF NOT EXISTS distributions (
CREATE TABLE distributions (
release_id smallint,
filename varchar(255),
url varchar(255),

79
tools/make-cheeses.py

@ -1,48 +1,63 @@
#!/usr/bin/env python3
from typing import KeysView
from typing import Any, Awaitable, KeysView
from asks.sessions import Session
from mysql.connector import connect
from trio import Nursery, open_nursery, run
PYPI = 'https://pypi.org'
USER = 'wensleydale'
DB = 'cheese_shop'
INDEX = 'https://pypi.org'
CONNECTIONS = 20
async def culture(session: Session) -> KeysView[str]:
"""Return the 100 most popular cheeses in cheese shop."""
stats = await session.get(
path='/stats', headers={'Accept': 'application/json'})
return stats.json()['top_packages'].keys()
async def drain(project_name: str, version: str, session: Session) -> None:
"""Fetch metadata of the given distribution."""
# XXX: insert the fetched metadata to a database
await session.get(path=f'/pypi/{project_name}/{version}/json')
print(project_name, version)
async def coagulate(project_name: str, nursery: Nursery,
session: Session) -> None:
"""Fetch project's available versions and the metadata of each."""
response = await session.get(path=f'/pypi/{project_name}/json')
for version in response.json()['releases'].keys():
# Recklessly filter out prereleases
for n in version.split('.'):
try:
int(n)
except ValueError:
break
else:
nursery.start_soon(drain, project_name, version, session)
class CheeseMaker:
def __init__(self, nursery: Nursery, session: Session) -> None:
self.nursery, self.session = nursery, session
self.cursor = connect(user=USER, database=DB).cursor()
def start_soon(self, async_fn: Awaitable, *args: Any) -> None:
"""Creates a child task, scheduling await async_fn(*args)."""
self.nursery.start_soon(async_fn, *args)
async def json(self, path: str) -> Any:
"""Return the JSON response to the given GET request."""
response = await self.session.get(
path=path, headers={'Accept': 'application/json'})
return response.json()
async def culture(self) -> KeysView[str]:
"""Return the 100 most popular cheeses in cheese shop."""
stats = await self.json('/stats')
return stats['top_packages'].keys()
async def drain(self, project_name: str, version: str) -> None:
"""Fetch metadata of the given distribution."""
# XXX: insert the fetched metadata to a database
await self.json(f'/pypi/{project_name}/{version}/json')
print(project_name, version)
async def coagulate(self, project_name: str) -> None:
"""Fetch project's available versions and metadata."""
response = await self.json(f'/pypi/{project_name}/json')
for version in response['releases'].keys():
# Recklessly filter out prereleases
for n in version.split('.'):
try:
int(n)
except ValueError:
break
else:
self.start_soon(self.drain, project_name, version)
async def main(session: Session):
"""Make cheeses."""
async with open_nursery() as nursery:
for project_name in await culture(session):
nursery.start_soon(coagulate, project_name, nursery, session)
maker = CheeseMaker(nursery, session)
for project_name in await maker.culture():
maker.start_soon(maker.coagulate, project_name)
if __name__ == '__main__':
run(main, Session(PYPI, connections=CONNECTIONS))
run(main, Session(INDEX, connections=CONNECTIONS))

1
tools/requirements.txt

@ -1,2 +1,3 @@
asks
mysql.connector
trio

2
tox.ini

@ -10,7 +10,7 @@ deps =
isort[requirements]
commands =
flake8
isort -c --diff
isort . --check --diff
[flake8]
hang-closing = True

Loading…
Cancel
Save