From ffc53b935ab0f1e7cf1c85e4bc57063bc99451f6 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sat, 23 Apr 2022 13:08:40 +0200 Subject: [PATCH] Finish migration to SQLAlchemy and automate testing cron.py --- Makefile | 29 +++++++++++++++++++-- gpg-mailgate-web/cron.py | 44 ++++++++++++++------------------ gpg-mailgate-web/schema.py | 20 --------------- test/gpg-mailgate-cron-test.conf | 27 ++++++++++++++++++++ test/schema.py | 37 +++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 47 deletions(-) delete mode 100644 gpg-mailgate-web/schema.py create mode 100644 test/gpg-mailgate-cron-test.conf create mode 100644 test/schema.py diff --git a/Makefile b/Makefile index 9c4ce7b..4fb9ad7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .POSIX: -.PHONY: test unittest pre-clean clean +.PHONY: test e2etest unittest crontest pre-clean clean # # On systems where Python 3.x binary has a different name, just @@ -12,6 +12,13 @@ # PYTHON = python3 +TEST_DB = test/lacre.db + +# +# Main goal to run tests. +# +test: e2etest unittest crontest + # # Run a set of end-to-end tests. # @@ -19,9 +26,27 @@ PYTHON = python3 # file. Basically this is just a script that feeds GPG Mailgate with # known input and checks whether output meets expectations. # -test: test/tmp test/logs pre-clean +e2etest: test/tmp test/logs pre-clean $(PYTHON) test/e2e_test.py +# +# Run a basic cron-job test. +# +# We use PYTHONPATH to make sure that cron.py can import GnuPG +# package. We also set GPG_MAILGATE_CONFIG env. variable to make sure +# it slurps the right config. +# +crontest: clean-db $(TEST_DB) + GPG_MAILGATE_CONFIG=test/gpg-mailgate-cron-test.conf PYTHONPATH=`pwd` $(PYTHON) gpg-mailgate-web/cron.py + +$(TEST_DB): + $(PYTHON) test/schema.py $(TEST_DB) + +# Before running the crontest goal we need to make sure that the +# database gets regenerated. +clean-db: + rm -f $(TEST_DB) + # # Run unit tests # diff --git a/gpg-mailgate-web/cron.py b/gpg-mailgate-web/cron.py index 292a1a2..44e7fe2 100644 --- a/gpg-mailgate-web/cron.py +++ b/gpg-mailgate-web/cron.py @@ -22,8 +22,7 @@ from configparser import RawConfigParser import GnuPG import sqlalchemy -from sqlalchemy.sql import select -# import MySQLdb +from sqlalchemy.sql import select, delete, update, and_ import smtplib import markdown import syslog @@ -37,6 +36,7 @@ from email.mime.multipart import MIMEMultipart CONFIG_PATH_ENV = "GPG_MAILGATE_CONFIG" def appendLog(msg): + print(msg) if 'logging' in cfg and 'file' in cfg['logging']: if cfg['logging'].get('file') == "syslog": syslog.syslog(syslog.LOG_INFO | syslog.LOG_MAIL, msg) @@ -110,53 +110,47 @@ if 'database' in cfg and 'enabled' in cfg['database'] and cfg['database']['enabl (engine, conn) = setup_db_connection(cfg["database"]["url"]) (gpgmw_keys) = define_db_schema() - # Original query: "SELECT publickey, id, email FROM gpgmw_keys WHERE status = 0 AND confirm = '' LIMIT 100" - # TODO: add WHERE clause referencing "status" and "confirm" columns. - query = select(gpgmw_keys.c.publickey, gpgmw_keys.c.id, gpgmw_keys.c.email).limit(100) - result_set = conn.execute(query) + selq = select(gpgmw_keys.c.publickey, gpgmw_keys.c.id, gpgmw_keys.c.email)\ + .where(and_(gpgmw_keys.c.status == 0, gpgmw_keys.c.confirm == ""))\ + .limit(100) + result_set = conn.execute(selq) - rowno = 0 for row in result_set: # delete any other public keys associated with this confirmed email address - # TODO: replace with a delete() query generator. - conn.execute("DELETE FROM gpgmw_keys WHERE email = %s AND id != %s", (row[2], row[1],)) + delq = delete(gpgmw_keys).where(and_(gpgmw_keys.c.email == row[2], gpgmw_keys.c.id != row[1])) + conn.execute(delq) GnuPG.delete_key(cfg['gpg']['keyhome'], row[2]) appendLog('Deleted key for <' + row[2] + '> via import request') if row[0].strip(): # we have this so that user can submit blank key to remove any encryption if GnuPG.confirm_key(row[0], row[2]): GnuPG.add_key(cfg['gpg']['keyhome'], row[0]) # import the key to gpg - # TODO: replace with an update() query generator. - conn.execute("UPDATE gpgmw_keys SET status = 1 WHERE id = %s", (row[1],)) # mark key as accepted + modq = gpgmw_keys.update().where(gpgmw_keys.c.id == row[1]).values(status = 1) + conn.execute(modq) # mark key as accepted appendLog('Imported key from <' + row[2] + '>') if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key registration successful", "registrationSuccess.md", row[2] ) else: - # TODO: replace with a delete() query generator. - conn.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) # delete key + delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1]) + conn.execute(delq) # delete key appendLog('Import confirmation failed for <' + row[2] + '>') if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key registration failed", "registrationError.md", row[2] ) else: # delete key so we don't continue processing it - # TODO: replace with a delete() query generator. - conn.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) + delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1]) + conn.execute(delq) if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key deleted", "keyDeleted.md", row[2]) - # connection.commit() - # TODO: see if it's still necessary - # delete keys - # TODO: replace with a delete() query generator. - result_set = conn.execute("SELECT email, id FROM gpgmw_keys WHERE status = 2 LIMIT 100") + stat2q = select(gpgmw_keys.c.email, gpgmw_keys.c.id).where(gpgmw_keys.c.status == 2).limit(100) + stat2_result_set = conn.execute(stat2q) - for row in result_set: + for row in stat2_result_set: GnuPG.delete_key(cfg['gpg']['keyhome'], row[0]) - # TODO: replace with a delete() query generator. - conn.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) + delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1]) + conn.execute(delq) appendLog('Deleted key for <' + row[0] + '>') - # connection.commit() - # TODO: see if it's still necessary else: print("Warning: doing nothing since database settings are not configured!") diff --git a/gpg-mailgate-web/schema.py b/gpg-mailgate-web/schema.py deleted file mode 100644 index 9bcad38..0000000 --- a/gpg-mailgate-web/schema.py +++ /dev/null @@ -1,20 +0,0 @@ -import sqlalchemy - -def define_db_schema(): - meta = sqlalchemy.MetaData() - - gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True), - sqlalchemy.Column('email', sqlalchemy.String(256)), - sqlalchemy.Column('publickey', sqlalchemy.Text), - sqlalchemy.Column('confirm', sqlalchemy.String(32)), - sqlalchemy.Column('status', sqlalchemy.Integer), - sqlalchemy.Column('time', sqlalchemy.DateTime)) - - return (meta, gpgmw_keys) - -(meta, gpgmw_keys) = define_db_schema() - -test_db = sqlalchemy.create_engine("sqlite:///test.db") - -meta.create_all(test_db) diff --git a/test/gpg-mailgate-cron-test.conf b/test/gpg-mailgate-cron-test.conf new file mode 100644 index 0000000..7f8ac0e --- /dev/null +++ b/test/gpg-mailgate-cron-test.conf @@ -0,0 +1,27 @@ +[logging] +config = test/gpg-lacre-log.ini +file = test/logs/gpg-mailgate.log +format = %(asctime)s %(module)s[%(process)d]: %(message)s +date_format = ISO + +[gpg] +keyhome = test/keyhome + +[smime] +cert_path = test/certs + +[database] +enabled = yes +url = sqlite:///test/lacre.db + +[relay] +host = localhost +port = 2500 + +[cron] +send_email = no + +[enc_keymap] +alice@disposlab = 1CD245308F0963D038E88357973CF4D9387C44D7 +bob@disposlab = 19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67 + diff --git a/test/schema.py b/test/schema.py new file mode 100644 index 0000000..b867ca0 --- /dev/null +++ b/test/schema.py @@ -0,0 +1,37 @@ +import sys +import sqlalchemy +from sqlalchemy.sql import insert + +def define_db_schema(): + meta = sqlalchemy.MetaData() + + gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta, + sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column('email', sqlalchemy.String(256)), + sqlalchemy.Column('publickey', sqlalchemy.Text), + sqlalchemy.Column('confirm', sqlalchemy.String(32)), + sqlalchemy.Column('status', sqlalchemy.Integer), + sqlalchemy.Column('time', sqlalchemy.DateTime)) + + return (meta, gpgmw_keys) + +if len(sys.argv) != 2: + print("ERROR: output database missing") + sys.exit(1) + +(meta, gpgmw_keys) = define_db_schema() + +dbname = sys.argv[1] +test_db = sqlalchemy.create_engine(f"sqlite:///{dbname}") + +# Initialise the schema +meta.create_all(test_db) + +conn = test_db.connect() + +# Populate the database with dummy data +conn.execute(gpgmw_keys.insert(), [ + {"id": 1, "email": "alice@lacre.io", "publickey": "PUBLICKEY1", "status": 0, "confirm": "", "time": None}, + {"id": 2, "email": "bob@lacre.io", "publickey": "PUBLICKEY2", "status": 0, "confirm": "", "time": None}, + {"id": 3, "email": "cecil@lacre.io", "publickey": "PUBLICKEY3", "status": 0, "confirm": "", "time": None} + ])