Merge pull request 'Fix minor cron.py issues' (#64) from fix-cron-script into master

Reviewed-on: #64
This commit is contained in:
pfm 2022-05-06 17:25:11 +00:00
commit 7767dfaff9
7 changed files with 254 additions and 33 deletions

View File

@ -71,25 +71,36 @@ def public_keys( keyhome ):
email = None
return keys
# confirms a key has a given email address
def to_bytes(s) -> bytes:
if isinstance(s, str):
return bytes(s, sys.getdefaultencoding())
else:
return s
# Confirms a key has a given email address by importing it into a temporary
# keyring. If this operation succeeds and produces a message mentioning the
# expected email, a key is confirmed.
def confirm_key( content, email ):
tmpkeyhome = ''
content = to_bytes(content)
expected_email = to_bytes(email.lower())
while True:
tmpkeyhome = '/tmp/' + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(12))
if not os.path.exists(tmpkeyhome):
break
os.mkdir(tmpkeyhome)
# let only the owner access the directory, otherwise gpg would complain
os.mkdir(tmpkeyhome, mode=0o700)
localized_env = os.environ.copy()
localized_env["LANG"] = "C"
p = subprocess.Popen( build_command(tmpkeyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=localized_env )
result = p.communicate(input=content)[1]
confirmed = False
for line in result.split("\n"):
if 'imported' in line and '<' in line and '>' in line:
if line.split('<')[1].split('>')[0].lower() == email.lower():
for line in result.split(b"\n"):
if b'imported' in line and b'<' in line and b'>' in line:
if line.split(b'<')[1].split(b'>')[0].lower() == expected_email:
confirmed = True
break
else:
@ -102,6 +113,8 @@ def confirm_key( content, email ):
# adds a key and ensures it has the given email address
def add_key( keyhome, content ):
if isinstance(content, str):
content = bytes(content, sys.getdefaultencoding())
p = subprocess.Popen( build_command(keyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
p.communicate(input=content)
p.wait()

View File

@ -1,5 +1,5 @@
.POSIX:
.PHONY: test unittest pre-clean clean
.PHONY: test e2etest unittest crontest pre-clean clean
#
# On systems where Python 3.x binary has a different name, just
@ -12,6 +12,13 @@
#
PYTHON = python3
TEST_DB = test/lacre.db
#
# Main goal to run tests.
#
test: e2etest unittest crontest
#
# Run a set of end-to-end tests.
#
@ -19,9 +26,27 @@ PYTHON = python3
# file. Basically this is just a script that feeds GPG Mailgate with
# known input and checks whether output meets expectations.
#
test: test/tmp test/logs pre-clean
e2etest: test/tmp test/logs pre-clean
$(PYTHON) test/e2e_test.py
#
# Run a basic cron-job test.
#
# We use PYTHONPATH to make sure that cron.py can import GnuPG
# package. We also set GPG_MAILGATE_CONFIG env. variable to make sure
# it slurps the right config.
#
crontest: clean-db $(TEST_DB)
GPG_MAILGATE_CONFIG=test/gpg-mailgate-cron-test.conf PYTHONPATH=`pwd` $(PYTHON) gpg-mailgate-web/cron.py
$(TEST_DB):
$(PYTHON) test/schema.py $(TEST_DB)
# Before running the crontest goal we need to make sure that the
# database gets regenerated.
clean-db:
rm -f $(TEST_DB)
#
# Run unit tests
#

View File

@ -6,10 +6,17 @@ feed some input to GPG Mailgate and inspect the output.
## Running tests
To run tests, use command `make test` or `make unittest`.
To run tests, use command `make test`.
Tests produce some helpful logs, so inspect contents of `test/logs` directory
if something goes wrong.
There are 3 types of tests:
* `make e2etest` -- they cover a complete Lacre flow, from feeding it with
an email to accepting its encrypted form;
* `make unittest` -- just small tests of small units of code;
* `make crontest` -- execute cron job with a SQLite database.
E2E tests (`make e2etest`) should produce some helpful logs, so inspect
contents of `test/logs` directory if something goes wrong.
If your system's Python binary isn't found in your `$PATH` or you want to use
a specific binary, use make's macro overriding: `make test

View File

@ -21,14 +21,22 @@
from configparser import RawConfigParser
import GnuPG
import MySQLdb
import sqlalchemy
from sqlalchemy.sql import select, delete, update, and_
import smtplib
import markdown
import syslog
from email.MIMEText import MIMEText
import os
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
# Environment variable name we read to retrieve configuration path. This is to
# enable non-root users to set up and run GPG Mailgate and to make the software
# testable.
CONFIG_PATH_ENV = "GPG_MAILGATE_CONFIG"
def appendLog(msg):
print(msg)
if 'logging' in cfg and 'file' in cfg['logging']:
if cfg['logging'].get('file') == "syslog":
syslog.syslog(syslog.LOG_INFO | syslog.LOG_MAIL, msg)
@ -37,75 +45,112 @@ def appendLog(msg):
logfile.write(msg + "\n")
logfile.close()
def load_file(name):
f = open(name)
data = f.read()
f.close()
return data
def authenticate_maybe(smtp):
if 'smtp' in cfg and 'enabled' in cfg['smtp'] and cfg['smtp']['enabled'] == 'true':
smtp.connect(cfg['smtp']['host'],cfg['smtp']['port'])
smtp.ehlo()
if 'starttls' in cfg['smtp'] and cfg['smtp']['starttls'] == 'true':
smtp.starttls()
smtp.ehlo()
smtp.login(cfg['smtp']['username'], cfg['smtp']['password'])
def send_msg( mailsubject, messagefile, recipients = None ):
mailbody = file( cfg['cron']['mail_templates'] + "/" + messagefile).read()
mailbody = load_file( cfg['cron']['mail_templates'] + "/" + messagefile)
msg = MIMEMultipart("alternative")
msg["From"] = cfg['cron']['notification_email']
msg["To"] = recipients
msg["Subject"] = mailsubject
msg.attach(MIMEText(mailbody, 'plain'))
msg.attach(MIMEText(markdown.markdown(mailbody), 'html'))
if 'relay' in cfg and 'host' in cfg['relay'] and 'enc_port' in cfg['relay']:
relay = (cfg['relay']['host'], int(cfg['relay']['enc_port']))
smtp = smtplib.SMTP(relay[0], relay[1])
authenticate_maybe(smtp)
smtp.sendmail( cfg['cron']['notification_email'], recipients, msg.as_string() )
else:
appendLog("Could not send mail due to wrong configuration")
def setup_db_connection(url):
engine = sqlalchemy.create_engine(url)
return (engine, engine.connect())
def define_db_schema():
meta = sqlalchemy.MetaData()
gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta,
sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
sqlalchemy.Column('email', sqlalchemy.String(256)),
sqlalchemy.Column('publickey', sqlalchemy.Text),
sqlalchemy.Column('confirm', sqlalchemy.String(32)),
sqlalchemy.Column('status', sqlalchemy.Integer),
sqlalchemy.Column('time', sqlalchemy.DateTime))
return (gpgmw_keys)
# Read configuration from /etc/gpg-mailgate.conf
_cfg = RawConfigParser()
_cfg.read('/etc/gpg-mailgate.conf')
_cfg.read(os.getenv(CONFIG_PATH_ENV, '/etc/gpg-mailgate.conf'))
cfg = dict()
for sect in _cfg.sections():
cfg[sect] = dict()
for (name, value) in _cfg.items(sect):
cfg[sect][name] = value
if 'database' in cfg and 'enabled' in cfg['database'] and cfg['database']['enabled'] == 'yes' and 'name' in cfg['database'] and 'host' in cfg['database'] and 'username' in cfg['database'] and 'password' in cfg['database']:
connection = MySQLdb.connect(host = cfg['database']['host'], user = cfg['database']['username'], passwd = cfg['database']['password'], db = cfg['database']['name'], port = 3306)
cursor = connection.cursor()
if 'database' in cfg and 'enabled' in cfg['database'] and cfg['database']['enabled'] == 'yes' and 'url' in cfg['database']:
(engine, conn) = setup_db_connection(cfg["database"]["url"])
(gpgmw_keys) = define_db_schema()
# import keys
cursor.execute("SELECT publickey, id, email FROM gpgmw_keys WHERE status = 0 AND confirm = '' LIMIT 100")
result_set = cursor.fetchall()
selq = select(gpgmw_keys.c.publickey, gpgmw_keys.c.id, gpgmw_keys.c.email)\
.where(and_(gpgmw_keys.c.status == 0, gpgmw_keys.c.confirm == ""))\
.limit(100)
result_set = conn.execute(selq)
for row in result_set:
# delete any other public keys associated with this confirmed email address
cursor.execute("DELETE FROM gpgmw_keys WHERE email = %s AND id != %s", (row[2], row[1],))
delq = delete(gpgmw_keys).where(and_(gpgmw_keys.c.email == row[2], gpgmw_keys.c.id != row[1]))
conn.execute(delq)
GnuPG.delete_key(cfg['gpg']['keyhome'], row[2])
appendLog('Deleted key for <' + row[2] + '> via import request')
if row[0].strip(): # we have this so that user can submit blank key to remove any encryption
if GnuPG.confirm_key(row[0], row[2]):
GnuPG.add_key(cfg['gpg']['keyhome'], row[0]) # import the key to gpg
cursor.execute("UPDATE gpgmw_keys SET status = 1 WHERE id = %s", (row[1],)) # mark key as accepted
modq = gpgmw_keys.update().where(gpgmw_keys.c.id == row[1]).values(status = 1)
conn.execute(modq) # mark key as accepted
appendLog('Imported key from <' + row[2] + '>')
if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes':
send_msg( "PGP key registration successful", "registrationSuccess.md", row[2] )
else:
cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) # delete key
delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1])
conn.execute(delq) # delete key
appendLog('Import confirmation failed for <' + row[2] + '>')
if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes':
send_msg( "PGP key registration failed", "registrationError.md", row[2] )
else:
# delete key so we don't continue processing it
cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],))
delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1])
conn.execute(delq)
if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes':
send_msg( "PGP key deleted", "keyDeleted.md", row[2])
connection.commit()
# delete keys
cursor.execute("SELECT email, id FROM gpgmw_keys WHERE status = 2 LIMIT 100")
result_set = cursor.fetchall()
stat2q = select(gpgmw_keys.c.email, gpgmw_keys.c.id).where(gpgmw_keys.c.status == 2).limit(100)
stat2_result_set = conn.execute(stat2q)
for row in result_set:
for row in stat2_result_set:
GnuPG.delete_key(cfg['gpg']['keyhome'], row[0])
cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],))
delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1])
conn.execute(delq)
appendLog('Deleted key for <' + row[0] + '>')
connection.commit()
else:
print("Warning: doing nothing since database settings are not configured!")

View File

@ -83,14 +83,27 @@ enc_port = 25
# Set this option to yes to use TLS for SMTP Servers which require TLS.
starttls = no
[smtp]
# Options when smtp auth is required to send out emails
enabled = false
username = gpg-mailgate
password = changeme
host = yourdomain.tld
port = 587
starttls = true
[database]
# uncomment the settings below if you want
# to read keys from a gpg-mailgate-web database
# TODO: see if this section is required by PHP. If not, delete it.
enabled = yes
name = gpgmw
host = localhost
username = gpgmw
password = password
# For other RDBMS backends, see:
# https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls
url = sqlite:///test.db
[enc_keymap]
# You can find these by running the following command:

View File

@ -0,0 +1,27 @@
[logging]
config = test/gpg-lacre-log.ini
file = test/logs/gpg-mailgate.log
format = %(asctime)s %(module)s[%(process)d]: %(message)s
date_format = ISO
[gpg]
keyhome = test/keyhome
[smime]
cert_path = test/certs
[database]
enabled = yes
url = sqlite:///test/lacre.db
[relay]
host = localhost
port = 2500
[cron]
send_email = no
[enc_keymap]
alice@disposlab = 1CD245308F0963D038E88357973CF4D9387C44D7
bob@disposlab = 19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67

91
test/schema.py Normal file
View File

@ -0,0 +1,91 @@
import sys
import sqlalchemy
from sqlalchemy.sql import insert
def define_db_schema():
meta = sqlalchemy.MetaData()
gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta,
sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
sqlalchemy.Column('email', sqlalchemy.String(256)),
sqlalchemy.Column('publickey', sqlalchemy.Text),
sqlalchemy.Column('confirm', sqlalchemy.String(32)),
sqlalchemy.Column('status', sqlalchemy.Integer),
sqlalchemy.Column('time', sqlalchemy.DateTime))
return (meta, gpgmw_keys)
if len(sys.argv) != 2:
print("ERROR: output database missing")
sys.exit(1)
(meta, gpgmw_keys) = define_db_schema()
dbname = sys.argv[1]
test_db = sqlalchemy.create_engine(f"sqlite:///{dbname}")
# Initialise the schema
meta.create_all(test_db)
conn = test_db.connect()
# Populate the database with dummy data
conn.execute(gpgmw_keys.insert(), [
{"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\
\
mQGNBGDYY5oBDAC+HAVjA05jsIpHfQ2KQ9m2olo1Qnlk+dkjD+Gagxj1ACezyiGL\
cfZfoE/MJYLCH9yPcX1fUIAPwdAyfJKlvkVcz+MhEpgl3aP3NM2L2unSx3v9ZFwT\
/qyMo9Zst5VSD04TVx2ySQB1vucd2ppgp66X7hlCxs+P8d0FV7VcdrNYol2oOtYP\
yEFXkdyXLI/INI6jrqNkBF87ej+dlTQZAm3zoj61Xwq4gW0YesAZoJyXs8X+a4Am\
8KF7YYcTcIy89yXflotmExpE+i77datSBLM/FpIPiUfkfK6q/TNyno8Z3PBC0QD5\
21leqfp/QHRkwmqFbIVuoeonCvrAccjM0ITLjW+P0xXJa3q0lQQCgcGOgqTuNWPT\
6FhlmvkXt6fBZ11C2I1b033HTePvjIwxOrEY8pSqYwerVX9EU7FXT+S98HNW/1nF\
cNk3SoofzUOcKZOwc5n0NEESrW7sWpmD6Qmf52+GURuO+15DSUt13xqmnte19Xqd\
n98y0wrYAUgyUY8AEQEAAbQPYWxpY2VAZGlzcG9zbGFiiQHUBBMBCAA+FiEEHNJF\
MI8JY9A46INXlzz02Th8RNcFAmDYY5oCGwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYC\
AwECHgECF4AACgkQlzz02Th8RNdZeAv+IVVK49f0tY5QOSERu5RqdyFNpsVlUws9\
swvSvXXK/ZQxZ3YD3o0WEJG5G8jRO+Zjrljx6zzH39ofEKn8QMQUuw+SVPrzbqQb\
Yp/idn1E9RZCyyhtwcYnIwUObq2NNsCk8UmnjYvpwoh/QcHic13/RSUj7vejujtB\
SRTjNUE/RK5ROY8r+xZW9ZV/Q0NEzKl2wQtmbt8vTRX9yNEB171XZHG7dg4bTzm+\
zs0jPGNT0ygcx+uE7DZ3RkyPLRk3fB+GPiYrL2lfPF1KkrHGY4PGhClKdR1kjfBA\
Kweb6ExZg0fBYlB8ia8z3RZQF29pztoVfk8KIimg9RoYNOKw3Jp5SnHsbz9JygmZ\
mp3M3Lrs7357oSn9x25/nrFGeUBWbbKoXSdoXZr0Ix4xxkOJPAK966w0pQq+sP+o\
Ozg3F2rFRc6SoQw1pNLQ57hhWTblQlz8ETY7GnVJ+0xiqkAq2hrLt0jhQ5taWjV6\
Fgy8fKUPd5OAMvB9bfmAErclWcqKarMcuQGNBGDYY5oBDAC6yOtgUwtKUsI3jTu2\
VdjNDEnt/VLdRseT4JosSMglZ963nlA4mltCjxj59DeM0Ft8eyF7Bu4EFw5Kid+O\
vKGA5rGZBE0IVROOvSJQNbcELkY9XYtZjOJ7elfG37rDQKfDk82xqod9iTd48nm7\
vrllvylQhKfXa+m99KxWabtKqCyXVjaZP9vfD3nVauu16oHW6rQavlLXo5MetFan\
Iwv1sTqnpzCt+cuG/7vUt89rOiJRalRP3/e1K5MSM6aWC/SHZs6HcrT+WT5nuPA+\
5VQ4gFCSb8UlscF4sI++hhB/k821vyl9hIjnR3aRiFWdrkykQOfZNhovvsnmJmk9\
+Zcq0M3pZBnBuLgxVwJNVa4gi63cYwtExpcAZcG28wSVmcXcPN2wxEpYg5n/nvvG\
8Dsk0AA5WU5WW8aLLLQNBmVg2y4Oa1Fy0M7yfSylLWBAdj7y8+UzspN6JCbYhOpP\
lLRCJv5+JOgR0MrA+lxfFZwfcSO12x+gkfQ9oyUBdXNuydMAEQEAAYkBtgQYAQgA\
IBYhBBzSRTCPCWPQOOiDV5c89Nk4fETXBQJg2GOaAhsMAAoJEJc89Nk4fETXpooL\
/iJKgNF80neUamewma1aZJjwKWoHysSWWSlPeU6pGctuJv15fbAfI/NM1iXnSEGt\
odsn0oHtuAASlVB0ckSFdE0a2DwLgO6s6oEJof/yrE5hIAAlwzjHsi1G/dtHcfIo\
SjHzE22qUZwwm5ketuvKvEDKKp3b1ccu37AZC1caRFh3q8xB5ByLh1gPiDJ+ehwU\
puXkXPdFQhQTZib4LYuMxzh6A+S9U0AM7WMKjX7PhJ68maOeQ+yOIBSWtBKyWwZu\
Sx01w+Y/USPz02AxUn102se52FCISc/NijlX1JvFQdzf/WaZu28nTmW9OXSW3WeK\
ql7zNQqj494JD8gJuRGCU9AaiCmOaBokRdLiGbin/wxiG1CkXGRDN5/r0m/1IoNz\
I4m2SLsB/a89WACQ//CKJyNn4xPOEQoix35tXjdjTLAVyTrX502vHGieZ3HJU2tb\
nmmMf/H0kReMtNYFwHxoTpBJ8vk+xcZ+6ETzH8nk6av+zZ/5T5Y0aD5zO89PcQk6\
pw==\
=Tbwz\
-----END PGP PUBLIC KEY BLOCK-----\
", "status": 0, "confirm": "", "time": None},
{"id": 2, "email": "bob@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\
\
mDMEYdTFkRYJKwYBBAHaRw8BAQdA2tgdP1pMt3cv3XAW7ov5AFn74mMZvyTksp9Q\
eO1PkpK0GkJvYiBGb29iYXIgPGJvYkBkaXNwb3NsYWI+iJYEExYIAD4WIQQZz0tH\
7MnEevqE1L2W85/aDjG7ZwUCYdTFkQIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgID\
AQIeAQIXgAAKCRCW85/aDjG7ZxVnAP49t7BU2H+/WCpa3fCAlMEcik82sU4p+U9D\
pMsbjawwYgEA1SbA5CF835cMjoEufy1h+2M4T9gI/0X2lk8OAtwwggm4OARh1MXg\
EgorBgEEAZdVAQUBAQdAUVNKx2OsGtNdRsnl3J/uv6obkUC0KcO4ikdRs+iejlMD\
AQgHiHgEGBYIACAWIQQZz0tH7MnEevqE1L2W85/aDjG7ZwUCYdTF4AIbDAAKCRCW\
85/aDjG7Z039APwLGP5ibqCC9yIr4YVbdWff1Ch+2C91MR2ObF93Up9+ogD8D2zd\
OjjB6xRD0Q2FN+alsNGCtdutAs18AZ5l33RMzws=\
=wWoq\
-----END PGP PUBLIC KEY BLOCK-----\
", "status": 0, "confirm": "", "time": None},
{"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 0, "confirm": "", "time": None}
])