diff --git a/GnuPG/__init__.py b/GnuPG/__init__.py index 8bcfebd..15a47af 100644 --- a/GnuPG/__init__.py +++ b/GnuPG/__init__.py @@ -71,25 +71,36 @@ def public_keys( keyhome ): email = None return keys -# confirms a key has a given email address +def to_bytes(s) -> bytes: + if isinstance(s, str): + return bytes(s, sys.getdefaultencoding()) + else: + return s + +# Confirms a key has a given email address by importing it into a temporary +# keyring. If this operation succeeds and produces a message mentioning the +# expected email, a key is confirmed. def confirm_key( content, email ): tmpkeyhome = '' + content = to_bytes(content) + expected_email = to_bytes(email.lower()) while True: tmpkeyhome = '/tmp/' + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(12)) if not os.path.exists(tmpkeyhome): break - os.mkdir(tmpkeyhome) + # let only the owner access the directory, otherwise gpg would complain + os.mkdir(tmpkeyhome, mode=0o700) localized_env = os.environ.copy() localized_env["LANG"] = "C" p = subprocess.Popen( build_command(tmpkeyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=localized_env ) result = p.communicate(input=content)[1] confirmed = False - for line in result.split("\n"): - if 'imported' in line and '<' in line and '>' in line: - if line.split('<')[1].split('>')[0].lower() == email.lower(): + for line in result.split(b"\n"): + if b'imported' in line and b'<' in line and b'>' in line: + if line.split(b'<')[1].split(b'>')[0].lower() == expected_email: confirmed = True break else: @@ -102,6 +113,8 @@ def confirm_key( content, email ): # adds a key and ensures it has the given email address def add_key( keyhome, content ): + if isinstance(content, str): + content = bytes(content, sys.getdefaultencoding()) p = subprocess.Popen( build_command(keyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.communicate(input=content) p.wait() diff --git a/Makefile b/Makefile index 9c4ce7b..4fb9ad7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .POSIX: -.PHONY: test unittest pre-clean clean +.PHONY: test e2etest unittest crontest pre-clean clean # # On systems where Python 3.x binary has a different name, just @@ -12,6 +12,13 @@ # PYTHON = python3 +TEST_DB = test/lacre.db + +# +# Main goal to run tests. +# +test: e2etest unittest crontest + # # Run a set of end-to-end tests. # @@ -19,9 +26,27 @@ PYTHON = python3 # file. Basically this is just a script that feeds GPG Mailgate with # known input and checks whether output meets expectations. # -test: test/tmp test/logs pre-clean +e2etest: test/tmp test/logs pre-clean $(PYTHON) test/e2e_test.py +# +# Run a basic cron-job test. +# +# We use PYTHONPATH to make sure that cron.py can import GnuPG +# package. We also set GPG_MAILGATE_CONFIG env. variable to make sure +# it slurps the right config. +# +crontest: clean-db $(TEST_DB) + GPG_MAILGATE_CONFIG=test/gpg-mailgate-cron-test.conf PYTHONPATH=`pwd` $(PYTHON) gpg-mailgate-web/cron.py + +$(TEST_DB): + $(PYTHON) test/schema.py $(TEST_DB) + +# Before running the crontest goal we need to make sure that the +# database gets regenerated. +clean-db: + rm -f $(TEST_DB) + # # Run unit tests # diff --git a/doc/testing.md b/doc/testing.md index d018770..98c7cec 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -6,10 +6,17 @@ feed some input to GPG Mailgate and inspect the output. ## Running tests -To run tests, use command `make test` or `make unittest`. +To run tests, use command `make test`. -Tests produce some helpful logs, so inspect contents of `test/logs` directory -if something goes wrong. +There are 3 types of tests: + + * `make e2etest` -- they cover a complete Lacre flow, from feeding it with + an email to accepting its encrypted form; + * `make unittest` -- just small tests of small units of code; + * `make crontest` -- execute cron job with a SQLite database. + +E2E tests (`make e2etest`) should produce some helpful logs, so inspect +contents of `test/logs` directory if something goes wrong. If your system's Python binary isn't found in your `$PATH` or you want to use a specific binary, use make's macro overriding: `make test diff --git a/gpg-mailgate-web/cron.py b/gpg-mailgate-web/cron.py index 98cd8d1..45fded7 100644 --- a/gpg-mailgate-web/cron.py +++ b/gpg-mailgate-web/cron.py @@ -21,14 +21,22 @@ from configparser import RawConfigParser import GnuPG -import MySQLdb +import sqlalchemy +from sqlalchemy.sql import select, delete, update, and_ import smtplib import markdown import syslog -from email.MIMEText import MIMEText +import os +from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart +# Environment variable name we read to retrieve configuration path. This is to +# enable non-root users to set up and run GPG Mailgate and to make the software +# testable. +CONFIG_PATH_ENV = "GPG_MAILGATE_CONFIG" + def appendLog(msg): + print(msg) if 'logging' in cfg and 'file' in cfg['logging']: if cfg['logging'].get('file') == "syslog": syslog.syslog(syslog.LOG_INFO | syslog.LOG_MAIL, msg) @@ -37,75 +45,112 @@ def appendLog(msg): logfile.write(msg + "\n") logfile.close() +def load_file(name): + f = open(name) + data = f.read() + f.close() + return data + +def authenticate_maybe(smtp): + if 'smtp' in cfg and 'enabled' in cfg['smtp'] and cfg['smtp']['enabled'] == 'true': + smtp.connect(cfg['smtp']['host'],cfg['smtp']['port']) + smtp.ehlo() + if 'starttls' in cfg['smtp'] and cfg['smtp']['starttls'] == 'true': + smtp.starttls() + smtp.ehlo() + smtp.login(cfg['smtp']['username'], cfg['smtp']['password']) + def send_msg( mailsubject, messagefile, recipients = None ): - mailbody = file( cfg['cron']['mail_templates'] + "/" + messagefile).read() + mailbody = load_file( cfg['cron']['mail_templates'] + "/" + messagefile) msg = MIMEMultipart("alternative") msg["From"] = cfg['cron']['notification_email'] msg["To"] = recipients msg["Subject"] = mailsubject - + msg.attach(MIMEText(mailbody, 'plain')) msg.attach(MIMEText(markdown.markdown(mailbody), 'html')) - + if 'relay' in cfg and 'host' in cfg['relay'] and 'enc_port' in cfg['relay']: relay = (cfg['relay']['host'], int(cfg['relay']['enc_port'])) smtp = smtplib.SMTP(relay[0], relay[1]) + authenticate_maybe(smtp) smtp.sendmail( cfg['cron']['notification_email'], recipients, msg.as_string() ) else: appendLog("Could not send mail due to wrong configuration") +def setup_db_connection(url): + engine = sqlalchemy.create_engine(url) + return (engine, engine.connect()) + +def define_db_schema(): + meta = sqlalchemy.MetaData() + + gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta, + sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column('email', sqlalchemy.String(256)), + sqlalchemy.Column('publickey', sqlalchemy.Text), + sqlalchemy.Column('confirm', sqlalchemy.String(32)), + sqlalchemy.Column('status', sqlalchemy.Integer), + sqlalchemy.Column('time', sqlalchemy.DateTime)) + + return (gpgmw_keys) + + # Read configuration from /etc/gpg-mailgate.conf _cfg = RawConfigParser() -_cfg.read('/etc/gpg-mailgate.conf') +_cfg.read(os.getenv(CONFIG_PATH_ENV, '/etc/gpg-mailgate.conf')) cfg = dict() for sect in _cfg.sections(): cfg[sect] = dict() for (name, value) in _cfg.items(sect): cfg[sect][name] = value -if 'database' in cfg and 'enabled' in cfg['database'] and cfg['database']['enabled'] == 'yes' and 'name' in cfg['database'] and 'host' in cfg['database'] and 'username' in cfg['database'] and 'password' in cfg['database']: - connection = MySQLdb.connect(host = cfg['database']['host'], user = cfg['database']['username'], passwd = cfg['database']['password'], db = cfg['database']['name'], port = 3306) - cursor = connection.cursor() +if 'database' in cfg and 'enabled' in cfg['database'] and cfg['database']['enabled'] == 'yes' and 'url' in cfg['database']: + (engine, conn) = setup_db_connection(cfg["database"]["url"]) + (gpgmw_keys) = define_db_schema() - # import keys - cursor.execute("SELECT publickey, id, email FROM gpgmw_keys WHERE status = 0 AND confirm = '' LIMIT 100") - result_set = cursor.fetchall() + selq = select(gpgmw_keys.c.publickey, gpgmw_keys.c.id, gpgmw_keys.c.email)\ + .where(and_(gpgmw_keys.c.status == 0, gpgmw_keys.c.confirm == ""))\ + .limit(100) + result_set = conn.execute(selq) for row in result_set: # delete any other public keys associated with this confirmed email address - cursor.execute("DELETE FROM gpgmw_keys WHERE email = %s AND id != %s", (row[2], row[1],)) + delq = delete(gpgmw_keys).where(and_(gpgmw_keys.c.email == row[2], gpgmw_keys.c.id != row[1])) + conn.execute(delq) GnuPG.delete_key(cfg['gpg']['keyhome'], row[2]) appendLog('Deleted key for <' + row[2] + '> via import request') if row[0].strip(): # we have this so that user can submit blank key to remove any encryption if GnuPG.confirm_key(row[0], row[2]): GnuPG.add_key(cfg['gpg']['keyhome'], row[0]) # import the key to gpg - cursor.execute("UPDATE gpgmw_keys SET status = 1 WHERE id = %s", (row[1],)) # mark key as accepted + modq = gpgmw_keys.update().where(gpgmw_keys.c.id == row[1]).values(status = 1) + conn.execute(modq) # mark key as accepted appendLog('Imported key from <' + row[2] + '>') if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key registration successful", "registrationSuccess.md", row[2] ) else: - cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) # delete key + delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1]) + conn.execute(delq) # delete key appendLog('Import confirmation failed for <' + row[2] + '>') if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key registration failed", "registrationError.md", row[2] ) else: # delete key so we don't continue processing it - cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) + delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1]) + conn.execute(delq) if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key deleted", "keyDeleted.md", row[2]) - connection.commit() - # delete keys - cursor.execute("SELECT email, id FROM gpgmw_keys WHERE status = 2 LIMIT 100") - result_set = cursor.fetchall() + stat2q = select(gpgmw_keys.c.email, gpgmw_keys.c.id).where(gpgmw_keys.c.status == 2).limit(100) + stat2_result_set = conn.execute(stat2q) - for row in result_set: + for row in stat2_result_set: GnuPG.delete_key(cfg['gpg']['keyhome'], row[0]) - cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) + delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1]) + conn.execute(delq) appendLog('Deleted key for <' + row[0] + '>') - connection.commit() else: print("Warning: doing nothing since database settings are not configured!") diff --git a/gpg-mailgate.conf.sample b/gpg-mailgate.conf.sample index 0b74a55..fad9a79 100644 --- a/gpg-mailgate.conf.sample +++ b/gpg-mailgate.conf.sample @@ -83,14 +83,27 @@ enc_port = 25 # Set this option to yes to use TLS for SMTP Servers which require TLS. starttls = no +[smtp] +# Options when smtp auth is required to send out emails +enabled = false +username = gpg-mailgate +password = changeme +host = yourdomain.tld +port = 587 +starttls = true + [database] # uncomment the settings below if you want # to read keys from a gpg-mailgate-web database +# TODO: see if this section is required by PHP. If not, delete it. enabled = yes name = gpgmw host = localhost username = gpgmw password = password +# For other RDBMS backends, see: +# https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls +url = sqlite:///test.db [enc_keymap] # You can find these by running the following command: diff --git a/test/gpg-mailgate-cron-test.conf b/test/gpg-mailgate-cron-test.conf new file mode 100644 index 0000000..7f8ac0e --- /dev/null +++ b/test/gpg-mailgate-cron-test.conf @@ -0,0 +1,27 @@ +[logging] +config = test/gpg-lacre-log.ini +file = test/logs/gpg-mailgate.log +format = %(asctime)s %(module)s[%(process)d]: %(message)s +date_format = ISO + +[gpg] +keyhome = test/keyhome + +[smime] +cert_path = test/certs + +[database] +enabled = yes +url = sqlite:///test/lacre.db + +[relay] +host = localhost +port = 2500 + +[cron] +send_email = no + +[enc_keymap] +alice@disposlab = 1CD245308F0963D038E88357973CF4D9387C44D7 +bob@disposlab = 19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67 + diff --git a/test/schema.py b/test/schema.py new file mode 100644 index 0000000..c0c40d7 --- /dev/null +++ b/test/schema.py @@ -0,0 +1,91 @@ +import sys +import sqlalchemy +from sqlalchemy.sql import insert + +def define_db_schema(): + meta = sqlalchemy.MetaData() + + gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta, + sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column('email', sqlalchemy.String(256)), + sqlalchemy.Column('publickey', sqlalchemy.Text), + sqlalchemy.Column('confirm', sqlalchemy.String(32)), + sqlalchemy.Column('status', sqlalchemy.Integer), + sqlalchemy.Column('time', sqlalchemy.DateTime)) + + return (meta, gpgmw_keys) + +if len(sys.argv) != 2: + print("ERROR: output database missing") + sys.exit(1) + +(meta, gpgmw_keys) = define_db_schema() + +dbname = sys.argv[1] +test_db = sqlalchemy.create_engine(f"sqlite:///{dbname}") + +# Initialise the schema +meta.create_all(test_db) + +conn = test_db.connect() + +# Populate the database with dummy data +conn.execute(gpgmw_keys.insert(), [ + {"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\ +\ +mQGNBGDYY5oBDAC+HAVjA05jsIpHfQ2KQ9m2olo1Qnlk+dkjD+Gagxj1ACezyiGL\ +cfZfoE/MJYLCH9yPcX1fUIAPwdAyfJKlvkVcz+MhEpgl3aP3NM2L2unSx3v9ZFwT\ +/qyMo9Zst5VSD04TVx2ySQB1vucd2ppgp66X7hlCxs+P8d0FV7VcdrNYol2oOtYP\ +yEFXkdyXLI/INI6jrqNkBF87ej+dlTQZAm3zoj61Xwq4gW0YesAZoJyXs8X+a4Am\ +8KF7YYcTcIy89yXflotmExpE+i77datSBLM/FpIPiUfkfK6q/TNyno8Z3PBC0QD5\ +21leqfp/QHRkwmqFbIVuoeonCvrAccjM0ITLjW+P0xXJa3q0lQQCgcGOgqTuNWPT\ +6FhlmvkXt6fBZ11C2I1b033HTePvjIwxOrEY8pSqYwerVX9EU7FXT+S98HNW/1nF\ +cNk3SoofzUOcKZOwc5n0NEESrW7sWpmD6Qmf52+GURuO+15DSUt13xqmnte19Xqd\ +n98y0wrYAUgyUY8AEQEAAbQPYWxpY2VAZGlzcG9zbGFiiQHUBBMBCAA+FiEEHNJF\ +MI8JY9A46INXlzz02Th8RNcFAmDYY5oCGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYC\ +AwECHgECF4AACgkQlzz02Th8RNdZeAv+IVVK49f0tY5QOSERu5RqdyFNpsVlUws9\ +swvSvXXK/ZQxZ3YD3o0WEJG5G8jRO+Zjrljx6zzH39ofEKn8QMQUuw+SVPrzbqQb\ +Yp/idn1E9RZCyyhtwcYnIwUObq2NNsCk8UmnjYvpwoh/QcHic13/RSUj7vejujtB\ +SRTjNUE/RK5ROY8r+xZW9ZV/Q0NEzKl2wQtmbt8vTRX9yNEB171XZHG7dg4bTzm+\ +zs0jPGNT0ygcx+uE7DZ3RkyPLRk3fB+GPiYrL2lfPF1KkrHGY4PGhClKdR1kjfBA\ +Kweb6ExZg0fBYlB8ia8z3RZQF29pztoVfk8KIimg9RoYNOKw3Jp5SnHsbz9JygmZ\ +mp3M3Lrs7357oSn9x25/nrFGeUBWbbKoXSdoXZr0Ix4xxkOJPAK966w0pQq+sP+o\ +Ozg3F2rFRc6SoQw1pNLQ57hhWTblQlz8ETY7GnVJ+0xiqkAq2hrLt0jhQ5taWjV6\ +Fgy8fKUPd5OAMvB9bfmAErclWcqKarMcuQGNBGDYY5oBDAC6yOtgUwtKUsI3jTu2\ +VdjNDEnt/VLdRseT4JosSMglZ963nlA4mltCjxj59DeM0Ft8eyF7Bu4EFw5Kid+O\ +vKGA5rGZBE0IVROOvSJQNbcELkY9XYtZjOJ7elfG37rDQKfDk82xqod9iTd48nm7\ +vrllvylQhKfXa+m99KxWabtKqCyXVjaZP9vfD3nVauu16oHW6rQavlLXo5MetFan\ +Iwv1sTqnpzCt+cuG/7vUt89rOiJRalRP3/e1K5MSM6aWC/SHZs6HcrT+WT5nuPA+\ +5VQ4gFCSb8UlscF4sI++hhB/k821vyl9hIjnR3aRiFWdrkykQOfZNhovvsnmJmk9\ ++Zcq0M3pZBnBuLgxVwJNVa4gi63cYwtExpcAZcG28wSVmcXcPN2wxEpYg5n/nvvG\ +8Dsk0AA5WU5WW8aLLLQNBmVg2y4Oa1Fy0M7yfSylLWBAdj7y8+UzspN6JCbYhOpP\ +lLRCJv5+JOgR0MrA+lxfFZwfcSO12x+gkfQ9oyUBdXNuydMAEQEAAYkBtgQYAQgA\ +IBYhBBzSRTCPCWPQOOiDV5c89Nk4fETXBQJg2GOaAhsMAAoJEJc89Nk4fETXpooL\ +/iJKgNF80neUamewma1aZJjwKWoHysSWWSlPeU6pGctuJv15fbAfI/NM1iXnSEGt\ +odsn0oHtuAASlVB0ckSFdE0a2DwLgO6s6oEJof/yrE5hIAAlwzjHsi1G/dtHcfIo\ +SjHzE22qUZwwm5ketuvKvEDKKp3b1ccu37AZC1caRFh3q8xB5ByLh1gPiDJ+ehwU\ +puXkXPdFQhQTZib4LYuMxzh6A+S9U0AM7WMKjX7PhJ68maOeQ+yOIBSWtBKyWwZu\ +Sx01w+Y/USPz02AxUn102se52FCISc/NijlX1JvFQdzf/WaZu28nTmW9OXSW3WeK\ +ql7zNQqj494JD8gJuRGCU9AaiCmOaBokRdLiGbin/wxiG1CkXGRDN5/r0m/1IoNz\ +I4m2SLsB/a89WACQ//CKJyNn4xPOEQoix35tXjdjTLAVyTrX502vHGieZ3HJU2tb\ +nmmMf/H0kReMtNYFwHxoTpBJ8vk+xcZ+6ETzH8nk6av+zZ/5T5Y0aD5zO89PcQk6\ +pw==\ +=Tbwz\ +-----END PGP PUBLIC KEY BLOCK-----\ +", "status": 0, "confirm": "", "time": None}, + {"id": 2, "email": "bob@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\ +\ +mDMEYdTFkRYJKwYBBAHaRw8BAQdA2tgdP1pMt3cv3XAW7ov5AFn74mMZvyTksp9Q\ +eO1PkpK0GkJvYiBGb29iYXIgPGJvYkBkaXNwb3NsYWI+iJYEExYIAD4WIQQZz0tH\ +7MnEevqE1L2W85/aDjG7ZwUCYdTFkQIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgID\ +AQIeAQIXgAAKCRCW85/aDjG7ZxVnAP49t7BU2H+/WCpa3fCAlMEcik82sU4p+U9D\ +pMsbjawwYgEA1SbA5CF835cMjoEufy1h+2M4T9gI/0X2lk8OAtwwggm4OARh1MXg\ +EgorBgEEAZdVAQUBAQdAUVNKx2OsGtNdRsnl3J/uv6obkUC0KcO4ikdRs+iejlMD\ +AQgHiHgEGBYIACAWIQQZz0tH7MnEevqE1L2W85/aDjG7ZwUCYdTF4AIbDAAKCRCW\ +85/aDjG7Z039APwLGP5ibqCC9yIr4YVbdWff1Ch+2C91MR2ObF93Up9+ogD8D2zd\ +OjjB6xRD0Q2FN+alsNGCtdutAs18AZ5l33RMzws=\ +=wWoq\ +-----END PGP PUBLIC KEY BLOCK-----\ +", "status": 0, "confirm": "", "time": None}, + {"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 0, "confirm": "", "time": None} + ])