Makefile

@@ -1,5 +1,6 @@
 .POSIX:
 .PHONY: test e2etest unittest crontest daemontest pre-clean clean restore-keyhome
+.SUFFIXES: .gv .png
 
 #
 # On systems where Python 3.x binary has a different name, just
@@ -12,6 +13,8 @@
 #
 PYTHON = python
 
+GRAPHVIZ = dot
+
 #
 # SQLite database used during tests
 #
@@ -20,11 +23,21 @@ PYTHON = python
 #
 TEST_DB = test/lacre.db
 
+#
+# List of graph files
+#
+GRAPHS = doc/key-lifecycle.png
+
 #
 # Main goal to run all tests.
 #
 test: e2etest daemontest unittest crontest
 
+#
+# Build graphviz diagrams.
+#
+doc: ${GRAPHS}
+
 #
 # Run a set of end-to-end tests.
 #
@@ -82,3 +95,7 @@ test/logs:
 
 clean: pre-clean clean-db
 	rm -rfv test/tmp test/logs
+
+# Convert dot source to PNG image.
+.gv.png:
+	$(GRAPHVIZ) -Tpng $< > ${<:S/.gv/.png/}

doc/key-lifecycle.gv

@@ -0,0 +1,76 @@
+digraph key_lifecycle {
+    node [fontname="Helvetica,Arial,sans-serif" fontsize=12 shape=Mrecord]
+    edge [fontname="Helvetica,Arial,sans-serif" fontsize=10]
+
+    start [label="" shape=circle]
+    end [label="" shape=circle]
+
+    // An ASCII-armoured key is stored in lacre_keys table with:
+    //
+    // lacre_keys.confirm = <random string>
+    // lacre_keys.status  = 0 (default value)
+    submitted [label="Submitted"]
+
+    // User has confirmed their email.
+    //
+    // lacre_keys.confirm = ''
+    confirmed [label="Email confirmed" color=green4]
+
+    // The key has been imported into GnuPG keyring and an identity has been
+    // created in lacre_identities table.
+    //
+    // lacre_keys.status = 1
+    imported [label="Imported" color=green4]
+
+    // Any old key for this email has been deleted.
+    deleted [label="Previous key\ndeleted"]
+
+    // When a key expires, we only fail to encrypt at the moment.
+    //
+    // See https://git.disroot.org/Disroot/gpg-lacre/issues/148
+    expired [label="Expired" color=red]
+
+    // A key may end up being non-usable in several different ways and this is
+    // a catch-all node to represent them.
+    //
+    // - User hasn't confirmed their email.
+    // - Provided key's email didn't match the one provided in submission form.
+    rejected [label="Key not used,\nremoved from database" color=brown]
+
+    // User submits ASCII-armoured OpenPGP key.
+    start -> submitted [label="user action:\nkey submission" color=green4]
+
+    // The user has clicked the confirmation link.
+    //
+    // - lacre_keys.confirm = ''
+    submitted -> confirmed [label="user action:\nemail confirmation" color=green4]
+
+    // Enough time has passed since submission that we decide to drop the key
+    // from the queue.
+    submitted -> rejected [label="confirmation timed out\nno user action" color=brown]
+
+    // A confirmed key is imported:
+    // - import into GnuPG keyring;
+    // - mark key as accepted (lacre_keys.status = 1);
+    // - update identity database;
+    // - send notification.
+    confirmed -> imported [label="import\n[non-empty key]" color=green4]
+
+    // Empty key is imported.
+    //
+    // Effectively this means key removal and disabling encryption.
+    confirmed -> deleted [label="import\n[empty key]" color=green4]
+    deleted -> end
+
+    // XXX: Import of revokation keys isn't implemented yet.
+    confirmed -> deleted [label="import\n[revokation key]\n(not implemented)" color=gray fontcolor=gray]
+
+    // Key validation fails, the key is not imported.
+    confirmed -> rejected [label="invalid key" color=brown]
+
+    // We don't explicitly make keys expired, but when they expire GnuPG
+    // refuses to encrypt payloads.
+    imported -> expired [label="expiry" color=red fontcolor=red]
+
+    rejected -> end
+}

lacre.conf.sample

@@ -133,6 +133,10 @@ pooling_mode = optimistic
 # made and closed after use, to avoid pool growth and connection rejections.
 #max_overflow = 10
 
+# Number of hours we will wait for the user to confirm their email.  Cron-job
+# will delete items older than this number of hours.  Default: 1h.
+#max_queue_hours = 1
+
 [enc_keymap]
 # You can find these by running the following command:
 #	gpg --list-keys --keyid-format long user@example.com

lacre.py

@@ -32,6 +32,7 @@ lacre.init_logging(conf.get_item('logging', 'config'))
 
 # This has to be executed *after* logging initialisation.
 import lacre.core as core
+from lacre.lazymessage import LazyMessage
 
 LOG = logging.getLogger('lacre.py')
 
@@ -45,14 +46,19 @@ def main():
             sys.exit(lacre.EX_CONFIG)
 
         delivered = False
-        try:
+        raw_message = None
+
+        # Read recipients from the command-line
+        to_addrs = sys.argv[1:]
+
         # Read e-mail from stdin, parse it
         raw = sys.stdin.read()
         raw_message = email.message_from_string(raw, policy=SMTPUTF8)
         from_addr = raw_message['From']
-            # Read recipients from the command-line
-            to_addrs = sys.argv[1:]
 
+        lmessage = LazyMessage(to_addrs, lambda: raw_message)
+
+        try:
             # Let's start
             core.deliver_message(raw_message, from_addr, to_addrs)
             delivered = True
@@ -64,6 +70,7 @@ def main():
             # some silly message-encoding issue that shouldn't bounce the
             # message, we just try recoding the message body and delivering it.
             try:
+                from_addr = raw_message['From']
                 core.failover_delivery(raw_message, to_addrs, from_addr)
             except:
                 LOG.exception('Failover delivery failed too')

lacre/__init__.py

@@ -43,7 +43,9 @@ EX_CONFIG = 78
 def init_logging(config_filename):
     if config_filename is not None:
         logging.config.fileConfig(config_filename)
+        logging.captureWarnings(True)
         logging.info('Configured from %s', config_filename)
     else:
         logging.config.dictConfig(FAIL_OVER_LOGGING_CONFIG)
+        logging.captureWarnings(True)
         logging.warning('Lacre logging configuration missing, using syslog as default')

lacre/config.py

@@ -6,6 +6,7 @@ configuration.
 
 from enum import Enum, auto
 from configparser import RawConfigParser
+from collections import namedtuple
 
 import os
 
@@ -128,9 +129,11 @@ def validate_config(*, additional=None):
 # High level access to configuration.
 #
 
-def relay_params():
-    """Return a (HOST, PORT) tuple identifying the mail relay."""
-    return (cfg["relay"]["host"], int(cfg["relay"]["port"]))
+Host = namedtuple('Host', ['name', 'port'])
+
+def relay_params() -> Host:
+    """Return a Host named tuple identifying the mail relay."""
+    return Host(name = cfg["relay"]["host"], port = int(cfg["relay"]["port"]))
 
 
 def daemon_params():

lacre/core.py

@@ -41,6 +41,7 @@ import lacre.recipients as recpt
 import lacre.smime as smime
 from lacre.transport import send_msg, register_sender, SendFrom
 from lacre.mailop import KeepIntact, InlineOpenPGPEncrypt, MimeOpenPGPEncrypt, MailSerialisationException
+from lacre.lazymessage import LazyMessage
 
 
 LOG = logging.getLogger(__name__)
@@ -117,7 +118,10 @@ def _sort_gpg_recipients(gpg_to) -> Tuple[recpt.RecipientList, recpt.RecipientLi
     return mime, inline
 
 
-def _gpg_encrypt_copy(message: EmailMessage, keys, recipients, encrypt_f):
+def _gpg_encrypt_copy(message: EmailMessage, keys, recipients, encrypt_f, lmessage: LazyMessage = None) -> EmailMessage:
+    if lmessage:
+        message = lmessage.get_message()
+
     msg_copy = copy.deepcopy(message)
     _customise_headers(msg_copy)
     encrypted_payloads = encrypt_f(msg_copy, keys)
@@ -125,8 +129,8 @@ def _gpg_encrypt_copy(message: EmailMessage, keys, recipients, encrypt_f):
     return msg_copy
 
 
-def _gpg_encrypt_to_bytes(message: EmailMessage, keys, recipients, encrypt_f) -> bytes:
-    msg_copy = _gpg_encrypt_copy(message, keys, recipients, encrypt_f)
+def _gpg_encrypt_to_bytes(message: EmailMessage, keys, recipients, encrypt_f, lmessage) -> bytes:
+    msg_copy = _gpg_encrypt_copy(message, keys, recipients, encrypt_f, lmessage)
     try:
         return msg_copy.as_bytes(policy=SMTPUTF8)
     except IndexError:
@@ -148,7 +152,9 @@ def _customise_headers(message: EmailMessage):
         message['X-Lacre'] = 'Encrypted by Lacre'
 
 
-def _encrypt_all_payloads_inline(message: EmailMessage, gpg_to_cmdline):
+def _encrypt_all_payloads_inline(message: EmailMessage, gpg_to_cmdline, lmessage: LazyMessage = None):
+    if lmessage:
+        message = lmessage.get_message()
 
     # This breaks cascaded MIME messages. Blame PGP/INLINE.
     encrypted_payloads = list()
@@ -164,7 +170,7 @@ def _encrypt_all_payloads_inline(message: EmailMessage, gpg_to_cmdline):
     return encrypted_payloads
 
 
-def _encrypt_all_payloads_mime(message: EmailMessage, gpg_to_cmdline):
+def _encrypt_all_payloads_mime(message: EmailMessage, gpg_to_cmdline, lmessage: LazyMessage = None):
     # Convert a plain text email into PGP/MIME attachment style.  Modeled after enigmail.
     pgp_ver_part = MIMEPart()
     pgp_ver_part.set_content('Version: 1' + text.EOL_S)
@@ -178,6 +184,9 @@ def _encrypt_all_payloads_mime(message: EmailMessage, gpg_to_cmdline):
     encrypted_part.set_param('inline', "",                'Content-Disposition')
     encrypted_part.set_param('filename', "encrypted.asc", 'Content-Disposition')
 
+    if lmessage:
+        message = lmessage.get_message()
+
     message.preamble = "This is an OpenPGP/MIME encrypted message (RFC 2440 and 3156)"
 
     boundary = _make_boundary()
@@ -201,12 +210,14 @@ def _encrypt_all_payloads_mime(message: EmailMessage, gpg_to_cmdline):
     return [pgp_ver_part, _encrypt_payload(encrypted_part, gpg_to_cmdline, check_nested)]
 
 
-def _rewrap_payload(message: EmailMessage) -> MIMEPart:
+def _rewrap_payload(message: EmailMessage, lmessage: LazyMessage = None) -> MIMEPart:
     # In PGP/MIME (RFC 3156), the payload has to be a valid MIME entity.  In
     # other words, we need to wrap text/* message's payload in a new MIME
     # entity.
 
     wrapper = MIMEPart(policy=SMTPUTF8)
+    if lmessage:
+        message = lmessage.get_message()
     content = message.get_content()
     wrapper.set_content(content)
 
@@ -234,7 +245,9 @@ def _set_type_and_boundary(message: EmailMessage, boundary):
     message.set_param('boundary', boundary)
 
 
-def _encrypt_payload(payload: EmailMessage, recipients, check_nested=True, **kwargs):
+def _encrypt_payload(payload: EmailMessage, recipients, check_nested=True, lmessage: LazyMessage = None, **kwargs):
+    if lmessage:
+        payload = lmessage.get_message()
     raw_payload = payload.get_payload(decode=True)
     LOG.debug('About to encrypt raw payload: %s', raw_payload)
     LOG.debug('Original message: %s', payload)
@@ -331,7 +344,9 @@ def failover_delivery(message: EmailMessage, recipients, from_address):
         LOG.warning('No failover strategy, giving up')
 
 
-def _is_encrypted(raw_message: EmailMessage):
+def _is_encrypted(raw_message: EmailMessage, lmessage: LazyMessage = None):
+    if lmessage:
+        raw_message = lmessage.get_message()
     if raw_message.get_content_type() == 'multipart/encrypted':
         return True
 
@@ -342,8 +357,11 @@ def _is_encrypted(raw_message: EmailMessage):
     return text.is_message_pgp_inline(first_part)
 
 
-def delivery_plan(recipients, message: EmailMessage, key_cache: kcache.KeyCache):
+def delivery_plan(recipients, message: EmailMessage, key_cache: kcache.KeyCache, lmessage: LazyMessage = None):
     """Generate a sequence of delivery strategies."""
+    if lmessage:
+        message = lmessage.get_message()
+
     if _is_encrypted(message):
         LOG.debug('Message is already encrypted: %s', message)
         return [KeepIntact(recipients)]

lacre/daemon.py

@@ -23,6 +23,7 @@ import lacre.core as gate
 import lacre.keyring as kcache
 import lacre.transport as xport
 from lacre.mailop import KeepIntact, MailSerialisationException
+from lacre.lazymessage import LazyMessage
 
 
 class MailEncryptionProxy:
@@ -37,37 +38,44 @@ class MailEncryptionProxy:
         with time_logger('Message delivery', LOG):
             try:
                 keys = self._keyring.freeze_identities()
+                lmessage = LazyMessage(envelope.rcpt_tos, lambda: envelope.original_content)
                 message = email.message_from_bytes(envelope.original_content, policy=SMTPUTF8)
 
                 if message.defects:
-                    LOG.warning("Issues found: %d; %s", len(message.defects), repr(message.defects))
+                    LOG.warning("Issues found: %s", repr(message.defects))
 
                 send = xport.SendFrom(envelope.mail_from)
-                for operation in gate.delivery_plan(envelope.rcpt_tos, message, keys):
+                for operation in gate.delivery_plan(envelope.rcpt_tos, message, keys, lmessage):
                     LOG.debug(f"Sending mail via {operation!r}")
                     try:
-                        new_message = operation.perform(message)
+                        new_message = operation.perform(message, lmessage)
                         send(new_message, operation.recipients())
-                    except (EncryptionException, MailSerialisationException, UnicodeEncodeError):
-                        # If the message can't be encrypted, deliver cleartext.
-                        LOG.exception('Unable to encrypt message, delivering in cleartext')
-                        if not isinstance(operation, KeepIntact):
+                    except (EncryptionException, MailSerialisationException) as e:
+                        # If the message can't be encrypted or serialised to a
+                        # stream of bytes, deliver original payload in
+                        # cleartext.
+                        LOG.error('Unable to encrypt message, delivering in cleartext: %s', e)
                         self._send_unencrypted(operation, envelope, send)
-                        else:
-                            LOG.exception('Cannot perform: %s', operation)
-                            raise
+
+            except xport.TransientFailure:
+                LOG.info('Bouncing message')
+                return xport.RESULT_TRANS_FAIL
+
+            except xport.PermanentFailure:
+                LOG.exception('Permanent failure')
+                return xport.RESULT_PERM_FAIL
 
             except:
+                if conf.should_log_headers():
+                    LOG.exception('Unexpected exception caught, bouncing message. Erroneous message headers: %s', self._beginning(envelope))
+                else:
                     LOG.exception('Unexpected exception caught, bouncing message')
 
-                if conf.should_log_headers():
-                    LOG.error('Erroneous message headers: %s', self._beginning(envelope))
-
-                return xport.RESULT_ERRORR
+                return xport.RESULT_TRANS_FAIL
 
         return xport.RESULT_OK
 
-    def _send_unencrypted(self, operation, envelope, send: xport.SendFrom):
+    def _send_unencrypted(self, operation, envelope: Envelope, send: xport.SendFrom):
         # Do not parse and re-generate the message, just send it as it is.
         send(envelope.original_content, operation.recipients())
 
@@ -80,9 +88,6 @@ class MailEncryptionProxy:
         end = min(limit, 2560)
         return e.original_content[0:end]
 
-    def _seconds_between(self, start_ms, end_ms) -> float:
-        return (end_ms - start_ms) * 1000
-
 
 def _init_controller(keys: kcache.KeyRing, max_body_bytes=None, tout: float = 5):
     proxy = MailEncryptionProxy(keys)
@@ -90,7 +95,10 @@ def _init_controller(keys: kcache.KeyRing, max_body_bytes=None, tout: float = 5)
     LOG.info(f"Initialising a mail Controller at {host}:{port}")
     return Controller(proxy, hostname=host, port=port,
                       ready_timeout=tout,
-                      data_size_limit=max_body_bytes)
+                      data_size_limit=max_body_bytes,
+                      # Do not decode data into str as we only operate on raw
+                      # data available via Envelope.original_content.
+                      decode_data=False)
 
 
 def _validate_config():

lacre/dbschema.py

@@ -16,7 +16,8 @@ import sqlalchemy
 # Values for lacre_keys.status column:
 # - ST_DEFAULT: initial state;
 # - ST_IMPORTED: key has been successfully processed by cron job;
-# - ST_TO_BE_DELETED: key can be deleted.
+# - ST_TO_BE_DELETED: key can be deleted.  We only have checks for this value
+#   but never assign it, so this is a candidate for removal.
 ST_DEFAULT, ST_IMPORTED, ST_TO_BE_DELETED = range(3)
 
 # lacre_keys.confirmed is set to an empty string when a key is confirmed by the user.

lacre/keymgmt.py

@@ -0,0 +1,16 @@
+"""Key management utilities."""
+
+from datetime import datetime, timedelta
+
+from lacre.config import get_item
+
+
+def calculate_expiry_date(now: datetime) -> datetime:
+    """Calculate date-time of key queue item expiry.
+
+    Given current timestamp and configuration item
+    [database]max_queue_hours, return a date-time object that should be
+    older than any key in our confirmation queue.  If a key is older
+    than this threshold, we should remove it."""
+    max_hours = get_item('database', 'max_queue_hours', 1)
+    return now - timedelta(hours=max_hours)

lacre/lazymessage.py

@@ -0,0 +1,33 @@
+from aiosmtpd.smtp import Envelope
+from email import message_from_bytes
+from email.message import EmailMessage
+from email.parser import BytesHeaderParser
+from email.policy import SMTPUTF8
+
+class LazyMessage:
+    def __init__(self, recipients, content_provider):
+        self._content_provider = content_provider
+        self._recipients = recipients
+        self._headers = None
+        self._message = None
+
+    def get_original_content(self) -> bytes:
+        return self._content_provider()
+
+    def get_recipients(self):
+        return self._recipients
+
+    def get_headers(self) -> EmailMessage:
+        if self._message:
+            return self._message
+
+        if not self._headers:
+            self._headers = BytesHeaderParser(policy=SMTPUTF8).parsebytes(self.get_original_content())
+
+        return self._headers
+
+    def get_message(self) -> EmailMessage:
+        if not self._message:
+            self._message = message_from_bytes(self.get_original_content(), policy=SMTPUTF8)
+
+        return self._message

lacre/mailop.py

@@ -15,7 +15,9 @@ There are 3 operations available:
 
 import logging
 import lacre.core as core
-from email.message import Message
+from lacre.lazymessage import LazyMessage
+from email.message import Message, EmailMessage
+from email.parser import BytesHeaderParser
 from email.policy import SMTP, SMTPUTF8
 
 
@@ -34,7 +36,7 @@ class MailOperation:
         """Initialise the operation with a recipient."""
         self._recipients = recipients
 
-    def perform(self, message: Message) -> bytes:
+    def perform(self, message: Message, lmessage: LazyMessage) -> bytes:
         """Perform this operation on MESSAGE.
 
         Return target message.
@@ -75,12 +77,13 @@ class InlineOpenPGPEncrypt(OpenPGPEncrypt):
         """Initialise strategy object."""
         super().__init__(recipients, keys, keyhome)
 
-    def perform(self, msg: Message) -> bytes:
+    def perform(self, msg: Message, lmessage: LazyMessage) -> bytes:
         """Encrypt with PGP Inline."""
         LOG.debug('Sending PGP/Inline...')
         return core._gpg_encrypt_to_bytes(msg,
                                           self._keys, self._recipients,
-                                          core._encrypt_all_payloads_inline)
+                                          core._encrypt_all_payloads_inline,
+                                          lmessage)
 
 
 class MimeOpenPGPEncrypt(OpenPGPEncrypt):
@@ -90,12 +93,13 @@ class MimeOpenPGPEncrypt(OpenPGPEncrypt):
         """Initialise strategy object."""
         super().__init__(recipients, keys, keyhome)
 
-    def perform(self, msg: Message) -> bytes:
+    def perform(self, msg: Message, lmessage: LazyMessage) -> bytes:
         """Encrypt with PGP MIME."""
         LOG.debug('Sending PGP/MIME...')
         return core._gpg_encrypt_to_bytes(msg,
                                           self._keys, self._recipients,
-                                          core._encrypt_all_payloads_mime)
+                                          core._encrypt_all_payloads_mime,
+                                          lmessage)
 
 
 class SMimeEncrypt(MailOperation):
@@ -107,7 +111,7 @@ class SMimeEncrypt(MailOperation):
         self._email = email
         self._cert = certificate
 
-    def perform(self, message: Message) -> bytes:
+    def perform(self, message: Message, lmessage: LazyMessage) -> bytes:
         """Encrypt with a certificate."""
         LOG.warning(f"Delivering clear-text to {self._recipients}")
         return message.as_bytes(policy=SMTP)
@@ -127,11 +131,11 @@ class KeepIntact(MailOperation):
         """Initialise pass-through operation for a given recipient."""
         super().__init__(recipients)
 
-    def perform(self, message: Message) -> bytes:
+    def perform(self, message: Message, lmessage: LazyMessage) -> bytes:
         """Return MESSAGE unmodified."""
         try:
-            return message.as_bytes(policy=SMTPUTF8)
-        except IndexError as e:
+            return lmessage.get_original_content()
+        except (IndexError, UnicodeEncodeError, ValueError) as e:
             raise MailSerialisationException(e)
 
     def __repr__(self):

lacre/notify.py

@@ -45,8 +45,8 @@ def notify(mailsubject, messagefile, recipients = None):
     msg.attach(MIMEText(markdown.markdown(mailbody), 'html'))
 
     if conf.config_item_set('relay', 'host') and conf.config_item_set('relay', 'enc_port'):
-        (host, port) = conf.relay_params()
-        smtp = smtplib.SMTP(host, port)
+        host = conf.relay_params()
+        smtp = smtplib.SMTP(host.name, host.port)
         _authenticate_maybe(smtp)
         LOG.info('Delivering notification: %s', recipients)
         smtp.sendmail(conf.get_item('cron', 'notification_email'), recipients, msg.as_string())

lacre/repositories.py

@@ -1,5 +1,7 @@
 """Lacre identity and key repositories."""
 
+from datetime import datetime, timedelta
+
 from sqlalchemy import create_engine, select, delete, and_, func
 from sqlalchemy.exc import OperationalError
 import logging
@@ -78,6 +80,7 @@ class IdentityRepository(KeyRing):
         LOG.debug('Registering identity: %s -- %s', insq, insq.compile().params)
         with self._engine.connect() as conn:
             conn.execute(insq)
+            conn.commit()
 
     def _update(self, email, fprint):
         upq = self._identities.update() \
@@ -87,6 +90,7 @@ class IdentityRepository(KeyRing):
         LOG.debug('Updating identity: %s -- %s', upq, upq.compile().params)
         with self._engine.connect() as conn:
             conn.execute(upq)
+            conn.commit()
 
     def delete(self, email):
         delq = delete(self._identities).where(self._identities.c.email == email)
@@ -94,6 +98,7 @@ class IdentityRepository(KeyRing):
 
         with self._engine.connect() as conn:
             conn.execute(delq)
+            conn.commit()
 
     def delete_all(self):
         LOG.warn('Deleting all identities from the database')
@@ -101,6 +106,7 @@ class IdentityRepository(KeyRing):
         delq = delete(self._identities)
         with self._engine.connect() as conn:
             conn.execute(delq)
+            conn.commit()
 
     def freeze_identities(self) -> KeyCache:
         """Return a static, async-safe copy of the identity map.
@@ -171,6 +177,22 @@ class KeyConfirmationQueue:
         with self._engine.connect() as conn:
             return [e for e in conn.execute(seldel)]
 
+    def delete_expired_queue_items(self, older_than: datetime):
+        """Remove keys that have been in queue before `older_than`."""
+        delq = delete(self._keys) \
+            .where(
+                and_(
+                     self._keys.c.time < older_than,
+                     # We only want to delete keys that haven't been confirmed.
+                     self._keys.c.confirm != db.CO_CONFIRMED
+                   )
+                )
+        LOG.debug('Deleting queue items older than %s: %s', repr(older_than), delq)
+
+        with self._engine.connect() as conn:
+            conn.execute(delq)
+            conn.commit()
+
     def delete_keys(self, row_id, /, email=None):
         """Remove key from the database."""
         if email is not None:
@@ -183,6 +205,7 @@ class KeyConfirmationQueue:
         with self._engine.connect() as conn:
             LOG.debug('Deleting public keys associated with confirmed email: %s', delq)
             conn.execute(delq)
+            conn.commit()
 
     def delete_key_by_email(self, email):
         """Remove keys linked to the given email from the database."""
@@ -191,6 +214,7 @@ class KeyConfirmationQueue:
         LOG.debug('Deleting email for: %s', email)
         with self._engine.connect() as conn:
             conn.execute(delq)
+            conn.commit()
 
     def mark_accepted(self, row_id):
         modq = self._keys.update().where(self._keys.c.id == row_id).values(status=db.ST_IMPORTED)
@@ -198,3 +222,4 @@ class KeyConfirmationQueue:
 
         with self._engine.connect() as conn:
             conn.execute(modq)
+            conn.commit()

lacre/transport.py

@@ -5,12 +5,19 @@ import logging
 from typing import AnyStr, List
 
 import lacre.config as conf
+from lacre.mailop import MailSerialisationException
 
 # Mail status constants.
 #
 # These are the only values that our mail handler is allowed to return.
 RESULT_OK = '250 OK'
-RESULT_ERROR = '500 Could not process your message'
+RESULT_TRANS_FAIL = '451 Aborted: error in processing'
+RESULT_PERM_FAIL = '554 Transaction failed'
+
+# See RFC 5321, section 4.2.1 "Reply Code Severities and Theory" for more
+# information on SMTP reply codes.
+RESP_TRANSIENT_NEG = 4
+RESP_PERMANENT_NEG = 5
 
 LOG = logging.getLogger(__name__)
 
@@ -34,7 +41,7 @@ def send_msg(message: AnyStr, recipients: List[str]):
     if recipients:
         LOG.info(f"Sending email to: {recipients!r}")
         relay = conf.relay_params()
-        smtp = smtplib.SMTP(relay[0], relay[1])
+        smtp = smtplib.SMTP(relay.name, relay.port)
         if conf.flag_enabled('relay', 'starttls'):
             smtp.starttls()
         smtp.sendmail(from_addr, recipients, message)
@@ -42,6 +49,19 @@ def send_msg(message: AnyStr, recipients: List[str]):
         LOG.info("No recipient found")
 
 
+class TransientFailure(BaseException):
+    """Signals a transient delivery failure (4xx SMTP reply).
+
+    Message should be bounced and re-sent later.
+    """
+    pass
+
+
+class PermanentFailure(BaseException):
+    """Signals a permanent delivery failure (5xx SMTP reply)."""
+    pass
+
+
 class SendFrom:
     """A class wrapping the transport process."""
 
@@ -63,9 +83,28 @@ class SendFrom:
 
         LOG.info("Sending email to: %s", recipients)
         relay = conf.relay_params()
-        smtp = smtplib.SMTP(relay[0], relay[1])
+        smtp = smtplib.SMTP(relay.name, relay.port)
 
         if conf.flag_enabled('relay', 'starttls'):
             smtp.starttls()
 
+        try:
             smtp.sendmail(self._from_addr, recipients, message)
+        except smtplib.SMTPResponseException as re:
+            resp_class = self._get_class(re.smtp_code)
+
+            if resp_class == RESP_TRANSIENT_NEG:
+                LOG.warning('Transient delivery failure: %s', re)
+                raise TransientFailure()
+            elif resp_class == RESP_PERMANENT_NEG:
+                LOG.error('Permanent delivery failure: %s', re)
+                raise PermanentFailure()
+        except smtplib.SMTPException as err:
+            LOG.error('Failed to deliver message: %s', err)
+            raise PermanentFailure()
+        except UnicodeEncodeError as uee:
+            LOG.error('Failed to deliver for non-SMTP reason', uee)
+            raise MailSerialisationException(uee)
+
+    def _get_class(self, resp_code):
+        return int(resp_code / 100)

requirements.txt

@@ -1,5 +1,5 @@
 aiosmtpd==1.4.2
-SQLAlchemy==1.4.32
+SQLAlchemy==2.0.29
 Markdown==3.4.1
 M2Crypto==0.38.0
 requests==2.27.1

test/daemon_test.py

@@ -23,15 +23,19 @@ import subprocess
 import os
 import time
 import unittest
+from typing import Dict
 
 
-def _spawn(cmd):
+def _spawn(cmd, *, env_add: Dict = None):
     env_dict = {
         "PATH": os.getenv("PATH"),
         "PYTHONPATH": os.getcwd(),
         "LANG": 'en_US.UTF-8',
         "LACRE_CONFIG": "test/lacre-daemon.conf"
         }
+    if env_add:
+        env_dict.update(env_add)
+
     logging.debug(f"Spawning command: {cmd} with environment: {env_dict!r}")
     return subprocess.Popen(cmd,
                             stdin=None,
@@ -45,7 +49,8 @@ def _interrupt(proc):
 
 def _send(host, port, mail_from, mail_to, message):
     logging.debug(f"Sending message to {host}:{port}")
-    p = _spawn([os.getenv("PYTHON") or "python",
+    python = os.getenv("PYTHON") or "python"
+    p = _spawn([python,
         "test/utils/sendmail.py",
         "-f", mail_from,
         "-t", mail_to,
@@ -83,7 +88,7 @@ class AdvancedMailFilterE2ETest(unittest.TestCase):
         python = os.getenv("PYTHON", "python")
 
         logging.info('Starting the server...')
-        cls.server = _spawn([python, '-m', 'lacre.daemon'])
+        cls.server = _spawn([python, '-m', 'lacre.daemon'], env_add={'SQLALCHEMY_WARN_20': '1'})
 
     @classmethod
     def tearDownClass(cls):

test/e2e.ini

@@ -137,7 +137,7 @@ out: -----BEGIN PGP MESSAGE-----
 descr: HTML, cleartext
 to: carlos@disposlab
 in: test/msgin/html-utf8.msg
-out: PGh0bWw+DQo8aGVhZD4NCjwvaGVhZD4NCjxib2R5Pg0KWkHFu8OTxYHEhiBHxJjFmkzEhCBKQcW5
+out: PGh0bWw+CjxoZWFkPgo8L2hlYWQ+Cjxib2R5PgpaQcW7w5PFgcSGIEfEmMWaTMSEIEpBxbnFgy48
 
 [case-17]
 descr: HTML, PGP/MIME

test/e2e_test.py

@@ -101,7 +101,7 @@ class SimpleMailFilterE2ETest(unittest.TestCase):
         cls._e2e_config_path = os.path.join(os.getcwd(), CONFIG_FILE)
 
         # This environment variable is set in Makefile.
-        cls._python_path = os.getenv('PYTHON', 'python3')
+        cls._python_path = os.getenv('PYTHON', 'python')
 
         _write_test_config(cls._e2e_config_path,
                            port				= cls._e2e_config.get("relay", "port"),

test/modules/test_contracts.py

@@ -186,6 +186,105 @@ class EmailParsingTest(unittest.TestCase):
         self.assertEqual(len(msg.defects), 0)
         self.assertRaises(IndexError, lambda: msg['Message-Id'])
 
+    def test_headersonly_text_plain(self):
+        rawmsg = b"From: alice@lacre.io\r\n" \
+          + b"To: bob@lacre.io\r\n" \
+          + b"Subject: Test message\r\n" \
+          + b"Content-Type: text/plain\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"Message-Id: <[yada-yada-yada@microsoft.com]>\r\n" \
+          + b"\r\n" \
+          + b"SGVsbG8sIFdvcmxkIQo=\r\n"
+
+        from email.parser import BytesHeaderParser
+        msg_headers_only = BytesHeaderParser(policy=SMTPUTF8).parsebytes(rawmsg)
+
+        self.assertEqual(msg_headers_only['From'], 'alice@lacre.io')
+        self.assertEqual(msg_headers_only.get_body().as_bytes(), rawmsg)
+        self.assertEqual(msg_headers_only.get_payload(), 'SGVsbG8sIFdvcmxkIQo=\r\n')
+
+    def test_headersonly_multipart_mixed(self):
+        rawmsg = b"From: eva@lacre.io\r\n" \
+          + b"Content-Type: multipart/mixed; boundary=XXXXXXXX\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX\r\n" \
+          + b"Content-Type: application/octet-stream\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX\r\n" \
+          + b"Content-Type: application/octet-stream\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"SGVsbG8sIFdvcmxkIQo=\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX--\r\n"
+
+        message_body = "--XXXXXXXX\r\n" \
+          + "Content-Type: application/octet-stream\r\n" \
+          + "Content-Transfer-Encoding: base64\r\n" \
+          + "\r\n" \
+          + "VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n" \
+          + "\r\n" \
+          + "--XXXXXXXX\r\n" \
+          + "Content-Type: application/octet-stream\r\n" \
+          + "Content-Transfer-Encoding: base64\r\n" \
+          + "\r\n" \
+          + "SGVsbG8sIFdvcmxkIQo=\r\n" \
+          + "\r\n" \
+          + "--XXXXXXXX--\r\n"
+
+        from email.parser import BytesHeaderParser
+        msg_headers_only = BytesHeaderParser(policy=SMTPUTF8).parsebytes(rawmsg)
+
+        self.assertEqual(msg_headers_only['From'], 'eva@lacre.io')
+        self.assertIsNone(msg_headers_only.get_body())
+        self.assertEqual(msg_headers_only.get_payload(), message_body)
+        self.assertRaises(KeyError, lambda: msg_headers_only.get_content())
+        self.assertFalse(msg_headers_only.is_multipart())
+
+    def test_headersonly_multipart_alternative(self):
+        rawmsg = b"From: eva@lacre.io\r\n" \
+          + b"Content-Type: multipart/alternative; boundary=XXXXXXXX\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX\r\n" \
+          + b"Content-Type: application/octet-stream\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX\r\n" \
+          + b"Content-Type: application/octet-stream\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"SGVsbG8sIFdvcmxkIQo=\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX--\r\n"
+
+        message_body = "--XXXXXXXX\r\n" \
+          + "Content-Type: application/octet-stream\r\n" \
+          + "Content-Transfer-Encoding: base64\r\n" \
+          + "\r\n" \
+          + "VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n" \
+          + "\r\n" \
+          + "--XXXXXXXX\r\n" \
+          + "Content-Type: application/octet-stream\r\n" \
+          + "Content-Transfer-Encoding: base64\r\n" \
+          + "\r\n" \
+          + "SGVsbG8sIFdvcmxkIQo=\r\n" \
+          + "\r\n" \
+          + "--XXXXXXXX--\r\n"
+
+        from email.parser import BytesHeaderParser
+        msg_headers_only = BytesHeaderParser(policy=SMTPUTF8).parsebytes(rawmsg)
+
+        self.assertEqual(msg_headers_only['From'], 'eva@lacre.io')
+        self.assertIsNone(msg_headers_only.get_body())
+        self.assertEqual(msg_headers_only.get_payload(), message_body)
+        self.assertRaises(KeyError, lambda: msg_headers_only.get_content())
+        self.assertFalse(msg_headers_only.is_multipart())
+
 
 class EmailTest(unittest.TestCase):
     def test_boundary_generated_after_as_string_call(self):

test/modules/test_keymgmt.py

@@ -0,0 +1,10 @@
+import unittest
+
+import datetime
+import lacre.keymgmt as km
+
+class KeyManagementUtilitiesTest(unittest.TestCase):
+    def test_expiry_date_calculation(self):
+        ts = datetime.datetime(2024, 1, 1, 12, 0)
+        exp = km.calculate_expiry_date(ts)
+        self.assertEqual(exp, datetime.datetime(2024, 1, 1, 11, 0))

test/utils/relay.py

@@ -101,6 +101,8 @@ def _serve(port) -> bytes:
 
     logging.debug('Received %d bytes of data', len(message))
 
+    s.close()
+
     # Trim EOM marker as we're only interested in the message body.
     return message[:-len(EOM)]
 

test/utils/schema.py

@@ -1,6 +1,5 @@
 import sys
 import sqlalchemy
-from sqlalchemy.sql import insert
 
 def define_db_schema():
     meta = sqlalchemy.MetaData()
@@ -25,13 +24,12 @@ if len(sys.argv) != 2:
 
 (meta, lacre_keys, identities) = define_db_schema()
 
-dbname = sys.argv[1]
-test_db = sqlalchemy.create_engine(f"sqlite:///{dbname}")
+test_db = sqlalchemy.create_engine(sqlalchemy.URL.create('sqlite', database=sys.argv[1]))
 
 # Initialise the schema
 meta.create_all(test_db)
 
-conn = test_db.connect()
+with test_db.connect() as conn:
 
     # Populate the database with dummy data
     conn.execute(lacre_keys.insert(), [
@@ -99,3 +97,5 @@ conn.execute(identities.insert(), [
         {'fingerprint': '19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67', 'email': 'bob@disposlab'},
         {'fingerprint': '530B1BB2D0CC7971648198BBA4774E507D3AF5BC', 'email': 'evan@disposlab'}
     ])
+
+    conn.commit()

test/utils/sendmail.py

@@ -2,6 +2,7 @@ import logging
 import smtplib
 import sys
 import getopt
+from contextlib import contextmanager
 from email import message_from_binary_file
 from email.policy import SMTPUTF8
 
@@ -16,10 +17,19 @@ def _load_message(name):
         return message_from_binary_file(f, policy=SMTPUTF8)
 
 
+@contextmanager
+def smtp_connection(host, port):
+    smtp = smtplib.SMTP(host, port)
+    try:
+        yield smtp
+    finally:
+        smtp.close()
+
+
 def _send_message(host, port, from_addr, recipients, message):
     logging.info(f"From {from_addr} to {recipients} at {host}:{port}")
     try:
-        smtp = smtplib.SMTP(host, port)
+        with smtp_connection(host, port) as smtp:
             return smtp.sendmail(from_addr, recipients, message.as_bytes())
     except smtplib.SMTPDataError as e:
         logging.error(f"Couldn't deliver message. Got error: {e}")
@@ -37,7 +47,7 @@ def _send_message(host, port, from_addr, recipients, message):
 # messages.
 def _send_bytes(host: str, port, from_addr: str, recipients, message: bytes):
     try:
-        smtp = smtplib.SMTP(host, port)
+        with smtp_connection(host, port) as smtp:
             smtp.ehlo_or_helo_if_needed()
             smtp.mail(from_addr)
             for r in recipients:

webgate-cron.py

@@ -20,10 +20,12 @@
 #
 
 import sys
+from datetime import datetime
 import logging
 import lacre
 import lacre.config as conf
 from lacre.notify import notify
+from lacre.keymgmt import calculate_expiry_date
 
 # Read configuration from /etc/lacre.conf
 conf.load_config()
@@ -56,7 +58,7 @@ def import_key(key_dir, armored_key, key_id, email, key_queue, identities):
 
 def import_failed(key_id, email, key_queue):
     key_queue.delete_keys(key_id)
-    LOG.warning('Import confirmation failed: %s', email)
+    LOG.warning('Key confirmation failed: %s', email)
 
     if conf.flag_enabled('cron', 'send_email'):
         notify("PGP key registration failed", "registrationError.md", email)
@@ -64,17 +66,16 @@ def import_failed(key_id, email, key_queue):
 
 def delete_key(key_id, email, key_queue):
     # delete key so we don't continue processing it
-    LOG.debug('Empty key received, just deleting')
+    LOG.debug('Empty key received, deleting known key from: %s', email)
 
-    key_queue.delete_keys(row_id)
+    key_queue.delete_keys(key_id, email)
     if conf.flag_enabled('cron', 'send_email'):
         notify("PGP key deleted", "keyDeleted.md", email)
 
 
 def cleanup(key_dir, key_queue):
     """Delete keys and queue entries."""
-
-    LOG.info('Cleaning up after a round of key confirmation')
+    LOG.debug('Removing no longer needed keys from queue')
     for email, row_id in key_queue.fetch_keys_to_delete():
         LOG.debug('Removing key from keyring: %s', email)
         GnuPG.delete_key(key_dir, email)
@@ -84,6 +85,9 @@ def cleanup(key_dir, key_queue):
 
         LOG.info('Deleted key for: %s', email)
 
+    expiry_date = calculate_expiry_date(datetime.now())
+    key_queue.delete_expired_queue_items(expiry_date)
+
 
 _validate_config()