From 3fcc8dd9980c9a50bdb9fa56047d78709a7b8de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Gia=20Phong?= Date: Sat, 4 Jul 2020 16:04:53 +0700 Subject: [PATCH] Refactor a little --- sql/ctl.sql | 12 +++++++ {tools => sql}/def.sql | 14 ++++----- tools/make-cheeses.py | 71 +++++++++++++++++++++++++----------------- tools/requirements.txt | 1 + tox.ini | 2 +- 5 files changed, 64 insertions(+), 36 deletions(-) create mode 100644 sql/ctl.sql rename {tools => sql}/def.sql (80%) diff --git a/sql/ctl.sql b/sql/ctl.sql new file mode 100644 index 0000000..4e146bb --- /dev/null +++ b/sql/ctl.sql @@ -0,0 +1,12 @@ +USE mysql; + +DROP user IF EXISTS wensleydale@localhost; + +CREATE USER wensleydale@localhost +IDENTIFIED BY ''; + +GRANT ALL PRIVILEGES ON cheese_shop.* +TO 'wensleydale'@'localhost'; + +UPDATE user SET plugin='mysql_native_password' WHERE User='wensleydale'; +FLUSH PRIVILEGES; diff --git a/tools/def.sql b/sql/def.sql similarity index 80% rename from tools/def.sql rename to sql/def.sql index af4713b..ce363b2 100644 --- a/tools/def.sql +++ b/sql/def.sql @@ -2,16 +2,16 @@ DROP DATABASE IF EXISTS cheese_shop; CREATE DATABASE cheese_shop; USE cheese_shop; -CREATE TABLE IF NOT EXISTS releases ( +CREATE TABLE releases ( id smallint AUTO_INCREMENT PRIMARY KEY, project varchar(32), version varchar(32)); -CREATE TABLE IF NOT EXISTS contacts ( +CREATE TABLE contacts ( email varchar(255) PRIMARY KEY, name varchar(255)); -CREATE TABLE IF NOT EXISTS information ( +CREATE TABLE information ( release_id smallint PRIMARY KEY, summary varchar(255), homepage varchar(2083), @@ -19,24 +19,24 @@ CREATE TABLE IF NOT EXISTS information ( FOREIGN KEY (release_id) REFERENCES releases(id), FOREIGN KEY (email) REFERENCES contacts(email)); -CREATE TABLE IF NOT EXISTS troves ( +CREATE TABLE troves ( id smallint AUTO_INCREMENT PRIMARY KEY, classifier varchar(255)); -CREATE TABLE IF NOT EXISTS classifiers ( +CREATE TABLE classifiers ( release_id smallint, trove_id smallint, PRIMARY KEY (release_id, trove_id), FOREIGN KEY (release_id) REFERENCES releases(id), FOREIGN KEY (trove_id) REFERENCES troves(id)); -CREATE TABLE IF NOT EXISTS keywords ( +CREATE TABLE keywords ( release_id smallint, term varchar(32), PRIMARY KEY (release_id, term), FOREIGN KEY (release_id) REFERENCES releases(id)); -CREATE TABLE IF NOT EXISTS distributions ( +CREATE TABLE distributions ( release_id smallint, filename varchar(255), url varchar(255), diff --git a/tools/make-cheeses.py b/tools/make-cheeses.py index b7327af..b8363a0 100755 --- a/tools/make-cheeses.py +++ b/tools/make-cheeses.py @@ -1,48 +1,63 @@ #!/usr/bin/env python3 -from typing import KeysView +from typing import Any, Awaitable, KeysView from asks.sessions import Session +from mysql.connector import connect from trio import Nursery, open_nursery, run -PYPI = 'https://pypi.org' +USER = 'wensleydale' +DB = 'cheese_shop' +INDEX = 'https://pypi.org' CONNECTIONS = 20 -async def culture(session: Session) -> KeysView[str]: - """Return the 100 most popular cheeses in cheese shop.""" - stats = await session.get( - path='/stats', headers={'Accept': 'application/json'}) - return stats.json()['top_packages'].keys() +class CheeseMaker: + def __init__(self, nursery: Nursery, session: Session) -> None: + self.nursery, self.session = nursery, session + self.cursor = connect(user=USER, database=DB).cursor() + def start_soon(self, async_fn: Awaitable, *args: Any) -> None: + """Creates a child task, scheduling await async_fn(*args).""" + self.nursery.start_soon(async_fn, *args) -async def drain(project_name: str, version: str, session: Session) -> None: - """Fetch metadata of the given distribution.""" - # XXX: insert the fetched metadata to a database - await session.get(path=f'/pypi/{project_name}/{version}/json') - print(project_name, version) + async def json(self, path: str) -> Any: + """Return the JSON response to the given GET request.""" + response = await self.session.get( + path=path, headers={'Accept': 'application/json'}) + return response.json() + async def culture(self) -> KeysView[str]: + """Return the 100 most popular cheeses in cheese shop.""" + stats = await self.json('/stats') + return stats['top_packages'].keys() -async def coagulate(project_name: str, nursery: Nursery, - session: Session) -> None: - """Fetch project's available versions and the metadata of each.""" - response = await session.get(path=f'/pypi/{project_name}/json') - for version in response.json()['releases'].keys(): - # Recklessly filter out prereleases - for n in version.split('.'): - try: - int(n) - except ValueError: - break - else: - nursery.start_soon(drain, project_name, version, session) + async def drain(self, project_name: str, version: str) -> None: + """Fetch metadata of the given distribution.""" + # XXX: insert the fetched metadata to a database + await self.json(f'/pypi/{project_name}/{version}/json') + print(project_name, version) + + async def coagulate(self, project_name: str) -> None: + """Fetch project's available versions and metadata.""" + response = await self.json(f'/pypi/{project_name}/json') + for version in response['releases'].keys(): + # Recklessly filter out prereleases + for n in version.split('.'): + try: + int(n) + except ValueError: + break + else: + self.start_soon(self.drain, project_name, version) async def main(session: Session): """Make cheeses.""" async with open_nursery() as nursery: - for project_name in await culture(session): - nursery.start_soon(coagulate, project_name, nursery, session) + maker = CheeseMaker(nursery, session) + for project_name in await maker.culture(): + maker.start_soon(maker.coagulate, project_name) if __name__ == '__main__': - run(main, Session(PYPI, connections=CONNECTIONS)) + run(main, Session(INDEX, connections=CONNECTIONS)) diff --git a/tools/requirements.txt b/tools/requirements.txt index 559c85b..8dc1aa9 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,2 +1,3 @@ asks +mysql.connector trio diff --git a/tox.ini b/tox.ini index 79488a6..9199842 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ deps = isort[requirements] commands = flake8 - isort -c --diff + isort . --check --diff [flake8] hang-closing = True