Merge pull request 'Improve error handling' (#146) from error-handling into main

Reviewed-on: #146

.gitignore

@@ -1,3 +1,6 @@
+# Generated project files:
+test/lacre.db
+
 *.py[cod]
 
 # C extensions
@@ -10,7 +13,6 @@ dist
 build
 eggs
 parts
-bin
 var
 sdist
 develop-eggs
@@ -26,10 +28,10 @@ pip-log.txt
 .tox
 nosetests.xml
 
-# GPG-Mailgate test files
+# Lacre test files
 test/logs
 test/tmp
-test/gpg-mailgate.conf
+test/lacre.conf
 test/keyhome/random_seed
 
 # Emacs files

GnuPG/__init__.py

@@ -1,20 +1,20 @@
 #
-# gpg-mailgate
+# lacre
 #
-# This file is part of the gpg-mailgate source code.
+# This file is part of the lacre source code.
 #
-# gpg-mailgate is free software: you can redistribute it and/or modify
+# lacre is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# gpg-mailgate source code is distributed in the hope that it will be useful,
+# lacre source code is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with gpg-mailgate source code. If not, see <http://www.gnu.org/licenses/>.
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
 #
 
 """GnuPG wrapper module."""
@@ -27,45 +27,100 @@ import random
 import string
 import sys
 import logging
+import re
+import tempfile
+from email.utils import parseaddr
 
 
 LINE_FINGERPRINT = 'fpr'
 LINE_USER_ID = 'uid'
+LINE_PUBLIC_KEY = 'pub'
 
 POS_FINGERPRINT = 9
+POS_UID = 9
 
 LOG = logging.getLogger(__name__)
 
+RX_CONFIRM = re.compile(br'key "([^"]+)" imported')
+
+
+class EncryptionException(Exception):
+    """Represents a failure to encrypt a payload.
+
+    Arguments passed to exception constructor:
+    - issue: human-readable explanation of the issue;
+    - recipient: owner of the key;
+    - cause: any additional information, if present;
+    - key: fingerprint of the key.
+    """
+    pass
+
 
 def _build_command(key_home, *args, **kwargs):
-    cmd = ["gpg", '--homedir', key_home] + list(args)
+    cmd = ["gpg", '--homedir', key_home]
+    cmd.extend(args)
     return cmd
 
 
-def public_keys(keyhome):
-    """List public keys from keyring KEYHOME."""
+def public_keys(keyhome, *, key_id=None):
+    """List public keys from keyring KEYHOME.
+
+    Returns a dict with fingerprints as keys and email as values."""
     cmd = _build_command(keyhome, '--list-keys', '--with-colons')
+    if key_id is not None:
+        cmd.append(key_id)
+
     p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     p.wait()
 
     keys = dict()
+    collected = set()
+
     fingerprint = None
     email = None
+
     for line in p.stdout.readlines():
         line = line.decode(sys.getdefaultencoding())
-        if line[0:3] == LINE_FINGERPRINT:
-            fingerprint = line.split(':')[POS_FINGERPRINT]
-        if line[0:3] == LINE_USER_ID:
-            if ('<' not in line or '>' not in line):
-                continue
-            email = line.split('<')[1].split('>')[0]
-        if not (fingerprint is None or email is None):
-            keys[fingerprint] = email
+
+        if line[0:3] == LINE_PUBLIC_KEY:
+            # New identity has started, reset state.
             fingerprint = None
             email = None
+
+        if line[0:3] == LINE_FINGERPRINT and not fingerprint:
+            fingerprint = _extract_fingerprint(line)
+
+        if line[0:3] == LINE_USER_ID:
+            email = _parse_uid_line(line)
+
+        if fingerprint and email and not email in collected:
+            keys[fingerprint] = email
+            collected.add(email)
+            fingerprint = None
+            email = None
+
+    p.stdout.close()
+    p.stderr.close()
     return keys
 
 
+def _extract_fingerprint(line):
+    fpr_line = line.split(':')
+    if len(fpr_line) <= POS_FINGERPRINT:
+        return None
+    else:
+        return fpr_line[POS_FINGERPRINT]
+
+
+def _parse_uid_line(line: str):
+    userid_line = line.split(':')
+    if len(userid_line) <= POS_UID:
+        return None
+    else:
+        (_, email) = parseaddr(userid_line[POS_UID])
+        return email
+
+
 def _to_bytes(s) -> bytes:
     if isinstance(s, str):
         return bytes(s, sys.getdefaultencoding())
@@ -76,32 +131,24 @@ def _to_bytes(s) -> bytes:
 # Confirms a key has a given email address by importing it into a temporary
 # keyring.  If this operation succeeds and produces a message mentioning the
 # expected email, a key is confirmed.
-def confirm_key(content, email):
+def confirm_key(content, email: str):
     """Verify that the key CONTENT is assigned to identity EMAIL."""
-    tmpkeyhome = ''
     content = _to_bytes(content)
-    expected_email = _to_bytes(email.lower())
+    expected_email = email.lower()
 
-    while True:
-        tmpkeyhome = '/tmp/' + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(12))
-        if not os.path.exists(tmpkeyhome):
-            break
+    tmpkeyhome = tempfile.mkdtemp()
+    LOG.debug('Importing into temporary directory: %s', tmpkeyhome)
 
-    # let only the owner access the directory, otherwise gpg would complain
-    os.mkdir(tmpkeyhome, mode=0o700)
-    localized_env = os.environ.copy()
-    localized_env["LANG"] = "C"
-    p = subprocess.Popen(_build_command(tmpkeyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=localized_env)
-    result = p.communicate(input=content)[1]
+    result = _import_key(tmpkeyhome, content)
     confirmed = False
 
-    for line in result.split(b"\n"):
-        if b'imported' in line and b'<' in line and b'>' in line:
-            if line.split(b'<')[1].split(b'>')[0].lower() == expected_email:
-                confirmed = True
-                break
-            else:
-                break # confirmation failed
+    for line in result.splitlines():
+        LOG.debug('Line from GnuPG: %s', line)
+        found = RX_CONFIRM.search(line)
+        if found:
+            (_, extracted_email) = parseaddr(found.group(1).decode())
+            confirmed = (extracted_email == expected_email)
+            LOG.debug('Confirmed email %s: %s', extracted_email, confirmed)
 
     # cleanup
     shutil.rmtree(tmpkeyhome)
@@ -109,27 +156,54 @@ def confirm_key(content, email):
     return confirmed
 
 
+def _import_key(keyhome, content):
+    content = _to_bytes(content)
+
+    # Ensure we get expected output regardless of the system locale.
+    localized_env = os.environ.copy()
+    localized_env["LANG"] = "C"
+
+    p = subprocess.Popen(_build_command(keyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=localized_env)
+    output = p.communicate(input=content)[1]
+    p.wait()
+
+    return output
+
+
 # adds a key and ensures it has the given email address
 def add_key(keyhome, content):
     """Register new key CONTENT in the keyring KEYHOME."""
-    if isinstance(content, str):
-        content = bytes(content, sys.getdefaultencoding())
-    p = subprocess.Popen(_build_command(keyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    p.communicate(input=content)
-    p.wait()
+    output = _import_key(keyhome, content)
+
+    email = None
+    for line in output.splitlines():
+        found = RX_CONFIRM.search(line)
+        if found:
+            (_, extracted_email) = parseaddr(found.group(1).decode())
+            email = extracted_email
+
+    # Find imported key to get its fingerprint
+    imported = public_keys(keyhome, key_id=email)
+
+    if len(imported.keys()) == 1:
+        fingerprint = list(imported.keys())[0]
+        return fingerprint, imported[fingerprint]
+    else:
+        return None, None
 
 
 def delete_key(keyhome, email):
     """Remove key assigned to identity EMAIL from keyring KEYHOME."""
-    from email.utils import parseaddr
     result = parseaddr(email)
 
     if result[1]:
         # delete all keys matching this email address
         p = subprocess.Popen(_build_command(keyhome, '--delete-key', '--batch', '--yes', result[1]), stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p.communicate()
         p.wait()
         return True
 
+    LOG.warn('Failed to parse email before deleting key: %s', email)
     return False
 
 
@@ -139,7 +213,7 @@ class GPGEncryptor:
     def __init__(self, keyhome, recipients=None, charset=None):
         """Initialise the wrapper."""
         self._keyhome = keyhome
-        self._message = b''
+        self._message = None
         self._recipients = list()
         self._charset = charset
         if recipients is not None:
@@ -147,16 +221,39 @@ class GPGEncryptor:
 
     def update(self, message):
         """Append MESSAGE to buffer about to be encrypted."""
-        self._message += message
+        if self._message is None:
+            self._message = message
+        else:
+            self._message += message
 
     def encrypt(self):
         """Feed GnuPG with the message."""
-        p = subprocess.Popen(self._command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        encdata = p.communicate(input=self._message)[0]
+        p = self._popen()
+        encdata, err = p.communicate(input=self._message)
+        if p.returncode != 0:
+            LOG.debug('Errors: %s', err)
+            details = parse_status(err)
+            raise EncryptionException(details['issue'], details['recipient'], details['cause'], details['key'])
         return (encdata, p.returncode)
 
+    def _popen(self):
+        if self._charset:
+            return subprocess.Popen(self._command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE,
+                                    encoding=self._charset)
+        else:
+            return subprocess.Popen(self._command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE)
+
     def _command(self):
-        cmd = _build_command(self._keyhome, "--trust-model", "always", "--batch", "--yes", "--pgp7", "--no-secmem-warning", "-a", "-e")
+        cmd = _build_command(self._keyhome,
+                             "--trust-model", "always",
+                             "--status-fd", "2",
+                             "--batch",
+                             "--yes",
+                             "--pgp7",
+                             "--no-secmem-warning",
+                             "-a", "-e")
 
         # add recipients
         for recipient in self._recipients:
@@ -168,7 +265,7 @@ class GPGEncryptor:
             cmd.append("--comment")
             cmd.append('Charset: ' + self._charset)
 
-        LOG.debug(f'Built command: {cmd!r}')
+        LOG.debug('Built command: %s', cmd)
         return cmd
 
 
@@ -192,3 +289,68 @@ class GPGDecryptor:
 
     def _command(self):
         return _build_command(self._keyhome, "--trust-model", "always", "--batch", "--yes", "--no-secmem-warning", "-a", "-d")
+
+
+STATUS_FD_PREFIX = b'[GNUPG:] '
+STATUS_FD_PREFIX_LEN = len(STATUS_FD_PREFIX)
+
+KEY_EXPIRED       = b'KEYEXPIRED'
+KEY_REVOKED       = b'KEYREVOKED'
+NO_RECIPIENTS     = b'NO_RECP'
+INVALID_RECIPIENT = b'INV_RECP'
+KEY_CONSIDERED    = b'KEY_CONSIDERED'
+NOAVAIL           = b'n/a'
+
+# INV_RECP reason code descriptions.
+INVALID_RECIPIENT_CAUSES = [
+    'No specific reason given',
+    'Not Found',
+    'Ambiguous specification',
+    'Wrong key usage',
+    'Key revoked',
+    'Key expired',
+    'No CRL known',
+    'CRL too old',
+    'Policy mismatch',
+    'Not a secret key',
+    'Key not trusted',
+    'Missing certificate',
+    'Missing issuer certificate',
+    'Key disabled',
+    'Syntax error in specification'
+    ]
+
+
+def parse_status(status_buffer: str) -> dict:
+    """Parse --status-fd output and return important information."""
+    return parse_status_lines(status_buffer.splitlines())
+
+
+def parse_status_lines(lines: list) -> dict:
+    """Parse --status-fd output and return important information."""
+    result = {'issue': NOAVAIL, 'recipient': NOAVAIL, 'cause': 'Unknown', 'key': NOAVAIL}
+
+    LOG.debug('Processing stderr lines %s', lines)
+
+    for line in lines:
+        LOG.debug('At gnupg stderr line %s', line)
+        if not line.startswith(STATUS_FD_PREFIX):
+            continue
+
+        if line.startswith(KEY_EXPIRED, STATUS_FD_PREFIX_LEN):
+            result['issue'] = 'key expired'
+        elif line.startswith(KEY_REVOKED, STATUS_FD_PREFIX_LEN):
+            result['issue'] = 'key revoked'
+        elif line.startswith(NO_RECIPIENTS, STATUS_FD_PREFIX_LEN):
+            result['issue'] = 'no recipients'
+        elif line.startswith(KEY_CONSIDERED, STATUS_FD_PREFIX_LEN):
+            result['key'] = line.split(b' ')[2]
+        elif line.startswith(INVALID_RECIPIENT, STATUS_FD_PREFIX_LEN):
+            words = line.split(b' ')
+            reason_code = int(words[2])
+            result['recipient'] = words[3]
+
+            if reason_code:
+                result['cause'] = INVALID_RECIPIENT_CAUSES[reason_code]
+
+    return result

INSTALL.md

@@ -1,185 +1,209 @@
 # Installation instructions
+
 ## Content
+
 - General information
-- Install GPG-Mailgate
-- Install GPG-Mailgate-Web
+- Install Lacre
+- Install [Lacre-Webgate](https://git.disroot.org/Lacre/lacre-webgate)
 - Install Register-handler
 
 ## General information
-GPG-Mailgate is divided in 3 main parts: GPG-Mailgate itself, GPG-Mailgate-Web and Register-handler. Some parts of the GPG-Mailgate project depend on other parts of the project. You will find information about these dependencies at the beginning of every installation part.
 
-These instructions show you how to set up GPG-Mailgate in an easy way. If you are a more advanced user, feel free to experiment with the settings. For these instructions a home directory for the user `nobody` is set. Sadly this is an odd workaround but no better solution was found.
+Lacre is divided in 3 main parts: Lacre itself, Lacre-Webgate and Register-handler. Some parts of the Lacre project depend on other parts of the project. You will find information about these dependencies at the beginning of every installation part.
+
+These instructions show you how to set up Lacre in an easy way. If you are a more advanced user, feel free to experiment with the settings. For these instructions a home directory for the user `nobody` is set. Sadly this is an odd workaround but no better solution was found.
 
 These instructions are based on an installation on an Ubuntu 14.04 LTS virtual machine. For other Linux distributions and other versions these instructions might need to be adapted to your distribution (e.g. installation of packages and used directories).
 
-## Install GPG-Mailgate
+## Install Lacre
+
 ### Requirements
-- Python 3.X is already installed
-- Postfix is already installed and configured. It is recommended that you have already tested your configuration so we can exclude this as a main cause of problems
-- GnuPG is already installed and configured
+
+- Python 3.9.
+- Dependencies listed in [requirements file](https://packaging.python.org/en/latest/tutorials/installing-packages/#requirements-files), `requirements.txt`.
+- Postfix: installed, configured and tested.
+- GnuPG: installed, configured and tested (e.g. via command-line).
 
 ### Installation
 
-1. Install the Python-M2Crypto module:
+1. Install the dependencies:
+
+```
+python -m pip install -r requirements.txt
+```
 
-        apt-get install python-m2crypto
-        
 2. Set the home directory for the user `nobody` (sadly this workaround is needed as there is no better solution at this point). If you get an error that the user is currently used by a process, you might need to kill the process manually.
 
-        usermod -d /var/gpgmailgate nobody
-        
+```
+usermod -d /var/gpgmailgate nobody
+```
+
 3. Create dedicated directories for storing PGP keys and S/MIME certificates and make the user `nobody` owner of these:
-        
-        mkdir -p /var/gpgmailgate/.gnupg
-        mkdir -p /var/gpgmailgate/smime
-        chown -R nobody:nogroup /var/gpgmailgate/
 
-4. Place the `gpg-mailgate.py` in `/usr/local/bin/`, make the user `nobody` owner of the file and make it executable:
+```
+install -u nobody -g nobody -d /var/gpgmailgate/ /var/gpgmailgate/.gnupg /var/gpgmailgate/smime
+```
 
-        chown nobody:nogroup /usr/local/bin/gpg-mailgate.py
-        chmod u+x /usr/local/bin/gpg-mailgate.py
+4. Place the `lacre.py` in `/usr/local/bin/`, make the user `nobody` owner of the file and make it executable:
 
-5. Place the `GnuPG` directory in `/usr/local/lib/python3.x/dist-packages` (replace 3.x with your Python version)
+```
+install -u nobody -g nobody -mode u=rx lacre.py /usr/local/bin/
+```
 
-6. Configure `/etc/gpg-mailgate.conf` based on the provided `gpg-mailgate.conf.sample`. Change the settings according to your configuration. If you follow this guide and have a standard configuration for postfix, you don't need to change much. 
+5. Place `GnuPG` and `lacre` directories in `/usr/local/lib/python3.x/dist-packages` (replace 3.x with your Python version).  Make sure they're available for Python `import`s by executing `python -m lacre.admin -h` command.
 
-7. Configure logging by copying `gpg-lacre-logging.conf.sample` to `/etc/gpg-lacre-logging.conf` and editing it according to your needs.  The path to this file is included in `[logging]` section of `gpg-mailgate.conf` file, so if you place it somewhere else, make sure to update the path too.  See also: [Configuration file format](https://docs.python.org/3/library/logging.config.html#configuration-file-format).
+6. Configure `/etc/lacre.conf` based on the provided `lacre.conf.sample`. Change the settings according to your configuration. If you follow this guide and have a standard configuration for postfix, you don't need to change much.
+
+7. Configure logging by copying `lacre-logging.conf.sample` to `/etc/lacre-logging.conf` and editing it according to your needs. The path to this file is included in `[logging]` section of `lacre.conf` file, so if you place it somewhere else, make sure to update the path too. See also: Python logging package's [Configuration file format](https://docs.python.org/3/library/logging.config.html#configuration-file-format).
 
 8. Add the following to the end of `/etc/postfix/master.cf`
 
-        gpg-mailgate    unix    -   n   n   -   -   pipe
-            flags= user=nobody argv=/usr/local/bin/gpg-mailgate.py ${recipient}
+```
+lacre    unix    -   n   n   -   -   pipe
+    flags= user=nobody argv=/usr/local/bin/lacre.py ${recipient}
 
-        127.0.0.1:10028 inet    n   -   n   -   10  smtpd
-            -o content_filter=
-            -o receive_override_options=no_unknown_recipient_checks,no_header_body_checks
-            -o smtpd_helo_restrictions=
-            -o smtpd_client_restrictions=
-            -o smtpd_sender_restrictions=
-            -o smtpd_recipient_restrictions=permit_mynetworks,reject
-            -o mynetworks=127.0.0.0/8
-            -o smtpd_authorized_xforward_hosts=127.0.0.0/8
+127. 0. 0. 1:10028 inet    n   -   n   -   10  smtpd
+    -o content_filter=
+    -o receive_override_options=no_unknown_recipient_checks,no_header_body_checks
+    -o smtpd_helo_restrictions=
+    -o smtpd_client_restrictions=
+    -o smtpd_sender_restrictions=
+    -o smtpd_recipient_restrictions=permit_mynetworks,reject
+    -o mynetworks=127. 0. 0. 0/8
+    -o smtpd_authorized_xforward_hosts=127. 0. 0. 0/8
+```
 
-    If you use Postfix versions from 2.5 onwards, it is recommended to change `${recipient}` to `${original_recipient}` in line two of the lines above.
+If you use Postfix versions from 2.5 onwards, it is recommended to change `${recipient}` to `${original_recipient}` in second line of the snippet above.
 
 9. Add the following line to `/etc/postfix/main.cf`
 
-        content_filter = gpg-mailgate
+```
+content_filter = lacre
+```
 
 10. Optional: GPG can automatically download new public keys for automatic signature verification. To enable automatic create the file `/var/gpgmailgate/.gnupg/gpg.conf`. Add the following line to the file:
 
-        keyserver-options auto-key-retrieve
+```
+keyserver-options auto-key-retrieve
+```
 
 11. Restart Postfix
 
 You are now ready to go. To add a public key for encryption just use the following command:
-        
-        sudo -u nobody /usr/bin/gpg --homedir=/var/gpgmailgate/.gnupg --import /some/public.key
 
-- Replace `/some/public.key` with the location of a public key
-- `/some/public.key` can be deleted after importation
-- Confirm that it's working:  
-`sudo -u nobody /usr/bin/gpg --list-keys --homedir=/var/gpgmailgate/.gnupg`
+```
+sudo -u nobody /usr/bin/gpg --homedir=/var/gpgmailgate/.gnupg --import /some/public.key
+```
+
+- Replace `/some/public.key` with the location of a public key (`/some/public.key` can be deleted after the import).
+- Confirm that it's working: `sudo -u nobody /usr/bin/gpg --list-keys --homedir=/var/gpgmailgate/.gnupg`
+
+If you already have a keyring you would like to import into Lacre, you can use `lacre.admin` command-line utility.  Read more in [Lacre administration](doc/admin.md).
 
 Please also test your installation before using it.
 
-GPG-Mailgate is also able to handle S/MIME certificates for encrypting mails. However, it is best to use it in combination with Register-Handler described later to add new certificates. If you try to add them manually it might fail. The certificates are stored in `/var/gpgmailgate/smime` in PKCS7 format and are named like `User@example.com` (the user part is case sensitive, the domain part should be in lower case).
+Lacre is also able to handle S/MIME certificates for encrypting mails. However, it is best to use it in combination with Register-Handler described later to add new certificates. If you try to add them manually it might fail. The certificates are stored in `/var/gpgmailgate/smime` in PKCS7 format and are named like `User@example.com` (the user part is case sensitive, the domain part should be in lower case).
+
+#### Additional settings
 
-####Additional settings
 Most mail servers do not handle mail addresses case sensitive. If you know that all your recipient mail servers do not care about case sensitivity then you can set `mail_case_insensitive` in the settings to `yes` so looking up PGP keys or S/MIME certificates does also happen case insensitive.
-If your recipients have problems to decrypt mails encrypted by GPG-Mailgate they might use a piece of software that does not support PGP/MIME encrypted mails. You can tell GPG-Mailgate to use the legacy PGP/INLINE format by adding the recipient to the `pgp_style` map in the following format:  
+If your recipients have problems to decrypt mails encrypted by Lacre they might use a piece of software that does not support PGP/MIME encrypted mails. You can tell Lacre to use the legacy PGP/INLINE format by adding the recipient to the `pgp_style` map in the following format:
 `User@example.com=inline`
 
 
-### Mail decryption
-GPG-Mailgate does not only feature encryption of mails but also decryption of PGP encrypted mails.
-#### Important notice
-**Read carefully before setting up and using this functionality!**
+## Install Lacre-Webgate
 
-With this functionality you could use GPG-Mailgate to decrypt incoming PGP encrypted mails (it is also capable of decrypting outgoing mails if the necessary key is present). To use this, you need to store your private keys on the server. This means that anyone who is able to obtain admin rights on the server is able to get the private keys stored on the server and is able to decrypt any mail encrypted with the corresponding public key. **If the server gets compromised in any kind and the attacker may have gained access to the server's file system, the keys have to be regarded as compromised as well!** If this happens you have to revoke your keys, notify everyone who has your public key (key servers as well) not to use this key any longer. You also need to create a new key pair for encrypted communication.
-
-#### Limitations
-There are two main types of PGP encryption: PGP/MIME and PGP/INLINE. PGP/MIME is standardized while PGP/INLINE isn't completely clear standardized (even though some people claim so). Decrypting PGP/MIME encrypted mails works in most cases while decrypting PGP/INLINE encrypted mails may fail more often. The reason is that most clients are implementing PGP/INLINE in their own way. GPG-Mailgate is able to decrypt mails which are encrypted PGP/INLINE by GPG-Mailgate on the sender's side. Furthermore it should be able to decrypt PGP/INLINE encrypted mails encrypted by Enigmail. For PGP/INLINE the mail's structure may not be preserved due to how PGP/INLINE is implemented on most clients. If you receive a PGP/INLINE encrypted mail that could not be decrypted by GPG-Mailgate you may ask the sender to use PGP/MIME instead. Furthermore file types might get lost when using PGP/INLINE. Due to this limitations decrypting PGP/INLINE encrypted mails is disabled by default. If you want to take the risk you can set `no_inline_dec` to `no` in the `[default]` section. You have been warned.
-
-#### Setting up decryption
-You need the recipient's private key for whom you want to decrypt mails. Only unprotected keys are supported. Keys protected by a passphrase could not be used. To add the private key, use the following command:  
-`sudo -u nobody /usr/bin/gpg --homedir=/var/gpgmailgate/.gnupg --import /some/private.key`
-From now on PGP encrypted mails will be decrypted for the recipients for whom the keys are imported.
-
-You also can remove a private key by using the following command. Replace `user@example.com` with the user's address for whom you want to remove the key:  
-`sudo -u nobody /usr/bin/gpg --homedir=/var/gpgmailgate/.gnupg --delete-secret-keys user@example.com`
-
-## Install GPG-Mailgate-Web
 ### Requirements
+
 - A webserver is installed and reachable
 - The webserver is able to handle PHP scripts
 - MySQL is installed
-- Python 3.X is already installed
+- Python 3.x is already installed
 
 ### Installation
-All files you need can be found in the [gpg-mailgate-web](gpg-mailgate-web/) directory.
+
+All files you need can be found in the
+[Lacre / lacre-webgate](https://git.disroot.org/Lacre/lacre-webgate/)
+repository.
 
 1. Install the Python-mysqldb and Python-markdown modules:
 
-        apt-get install python-mysqldb python-markdown
+```
+apt-get install python-mysqldb python-markdown
+```
 
-2. Create a new database for GPG-Mailgate-Web.
+2. Create a new database for Lacre-Webgate.
 
 3. Import the schema file `schema.sql` into the newly created database.
 
-4. Edit the config file located at `/etc/gpg-mailgate.conf`. Set `enabled = yes` in `[database]` and fill in the necessary settings for the database connection.
+4. Edit the config file located at `/etc/lacre.conf`. Set `enabled = yes` in `[database]` and fill in the necessary settings for the database connection.
 
-5. Copy the files located in the [public_html](gpg-mailgate-web/public_html) directory onto your webserver. They can also be placed in a subdirectory on your webserver.
+5. Copy the files located in the [public_html](https://git.disroot.org/Lacre/lacre-webgate/src/branch/main/public_html) directory onto your webserver. They can also be placed in a subdirectory on your webserver.
 
 6. On your webserver move the `config.sample.php` file to `config.php` and edit the configuration file.
 
 7. Create directories for storing email templates:
-        
-        mkdir -p /var/gpgmailgate/cron_templates
-        
+
+```
+mkdir -p /var/gpgmailgate/cron_templates
+```
+
 8. Copy the templates found in the [cron_templates](cron_templates/) directory into the newly created directory and transfer ownership:
 
-        chown -R nobody:nogroup /var/gpgmailgate/cron_templates
+```
+chown -R nobody:nogroup /var/gpgmailgate/cron_templates
+```
 
-9. Copy `cron.py` to `/usr/local/bin/gpgmw-cron.py`. Make it executable and and transfer ownership to `nobody`:
+9. Copy `cron.py` to `/usr/local/bin/cron.py`. Make it executable and and transfer ownership to `nobody`:
 
-        chown nobody:nogroup /usr/local/bin/gpgmw-cron.py
-        chmod u+x /usr/local/bin/gpgmw-cron.py
+```
+install -u nobody -g nobody -m u+x cron.py /usr/local/bin/lacre-cron.py
+```
 
-10. Create `/etc/cron.d/gpgmw` with contents:  
-`*/3 * * * * nobody /usr/bin/python /usr/local/bin/gpgmw-cron.py > /dev/null`  
+10. Create `/etc/cron.d/lacre-cron` with contents:
+`*/3 * * * * nobody /usr/bin/python /usr/local/bin/lacre-cron.py > /dev/null`
  for executing the cron job automatically.
 
 11. Test your installation.
 
-### GPG-Mailgate-Web as keyserver
-GPG-Mailgate-Web can also be used as a keyserver. For more information have a look at GPG-Mailgate-Web's [readme](gpg-mailgate-web/README).
+### Lacre-Webgate as keyserver
+
+Lacre-Webgate can also be used as a keyserver. For more information have a look at Lacre-Webgate's [README](https://git.disroot.org/Lacre/lacre-webgate/src/branch/main/README.md).
 
 ## Install Register-handler
+
 ### Requirements
-- Already set up and working GPG-Mailgate-Web. It should be reachable from the machine that will run register-handler
+
+- Already set up and working Lacre-Webgate. It should be reachable from the machine that will run register-handler
 - Postfix is already installed and configured. It is recommended that you have already tested your configuration so we can exclude this as a main cause of problems. Your Postfix configuration should also support aliases
 
 ### Installation
 
 1. Install the Python-requests module:
 
-        apt-get install python-requests
-        
+```
+apt-get install python-requests
+```
+
 2. Create directories for storing email templates:
-        
-        mkdir -p /var/gpgmailgate/register_templates
-        
+
+```
+mkdir -p /var/gpgmailgate/register_templates
+```
+
 3. Copy the templates found in the [register_templates](register_templates/) directory into the newly created directory and transfer ownership:
 
-        chown -R nobody:nogroup /var/gpgmailgate/register_templates
-        
+```
+chown -R nobody:nogroup /var/gpgmailgate/register_templates
+```
+
 4. Copy `register-handler.py` to `/usr/local/bin/register-handler.py`. Make it executable and own it to `nobody`:
 
-        chown nobody:nogroup /usr/local/bin/register-handler.py
-        chmod a+x /usr/local/bin/register-handler.py
-        
-5. Edit the config file located at `/etc/gpg-mailgate.conf`. Set the parameter `webpanel_url` in `[mailregister]` to the url of your GPG-Mailgate-Web panel (the URL should be the same as the one you use to access the panel with your web browser). Also set the parameter `register_email` to the email address you want the user to see when receiving mails from the register-handler (it does not have to be an existing address but it is recommended). Register-handler will send users mails when they are registering S/MIME certificates or when neither a S/MIME certificate nor a PGP key was found in a mail sent to the register-handler.
+```
+install -u nobody -g nogroup -m a+x register-handler.py /usr/local/bin/
+```
+
+5. Edit the config file located at `/etc/lacre.conf`. Set the parameter `webpanel_url` in `[mailregister]` to the url of your Lacre-Webgate panel (the URL should be the same as the one you use to access the panel with your web browser). Also set the parameter `register_email` to the email address you want the user to see when receiving mails from the register-handler (it does not have to be an existing address but it is recommended). Register-handler will send users mails when they are registering S/MIME certificates or when neither a S/MIME certificate nor a PGP key was found in a mail sent to the register-handler.
 
 6. Add `register:   |/usr/local/bin/register-handler.py` to `/etc/aliases`
 

Makefile

@@ -7,24 +7,30 @@
 #
 #     make test PYTHON=/usr/local/bin/python3.8
 #
-# This marco is passed via environment to test/e2e_test.py, where it's
+# This macro is passed via environment to test/e2e_test.py, where it's
 # used to compute further commands.
 #
-PYTHON = python3
+PYTHON = python
 
+#
+# SQLite database used during tests
+#
+# This database stores key queue and identity repository for e2etest,
+# daemontest, and crontest.
+#
 TEST_DB = test/lacre.db
 
 #
-# Main goal to run tests.
+# Main goal to run all tests.
 #
-test: e2etest unittest daemontest crontest
+test: e2etest daemontest unittest crontest
 
 #
 # Run a set of end-to-end tests.
 #
-# Test scenarios are described and configured by the test/e2e.ini
-# file.  Basically this is just a script that feeds GPG Mailgate with
-# known input and checks whether output meets expectations.
+# Test scenarios are described and configured by the test/e2e.ini file.
+# Basically this is just a script that feeds Lacre with known input and checks
+# whether output meets expectations.
 #
 e2etest: test/tmp test/logs pre-clean restore-keyhome
 	$(PYTHON) test/e2e_test.py
@@ -33,11 +39,12 @@ e2etest: test/tmp test/logs pre-clean restore-keyhome
 # Run a basic cron-job test.
 #
 # We use PYTHONPATH to make sure that cron.py can import GnuPG
-# package.  We also set GPG_MAILGATE_CONFIG env. variable to make sure
+# package.  We also set LACRE_CONFIG env. variable to make sure
 # it slurps the right config.
 #
 crontest: clean-db $(TEST_DB)
-	GPG_MAILGATE_CONFIG=test/gpg-mailgate-cron-test.conf PYTHONPATH=`pwd` $(PYTHON) webgate-cron.py
+	LACRE_CONFIG=test/lacre-daemon.conf PYTHONPATH=`pwd`	\
+	$(PYTHON) webgate-cron.py
 
 $(TEST_DB):
 	$(PYTHON) test/utils/schema.py $(TEST_DB)
@@ -45,7 +52,7 @@ $(TEST_DB):
 #
 # Run an e2e test of Advanced Content Filter.
 #
-daemontest:
+daemontest: restore-keyhome
 	$(PYTHON) test/daemon_test.py
 
 # Before running the crontest goal we need to make sure that the
@@ -57,14 +64,15 @@ clean-db:
 # Run unit tests
 #
 unittest:
-	$(PYTHON) -m unittest discover -s test/modules
+	LACRE_CONFIG=test/lacre.conf $(PYTHON) -m unittest discover -s test/modules
 
 pre-clean:
-	rm -fv test/gpg-mailgate.conf
+	rm -fv test/lacre.conf
 	rm -f test/logs/*.log
 
 restore-keyhome:
 	git restore test/keyhome
+	git restore test/keyhome.other
 
 test/tmp:
 	mkdir test/tmp
@@ -72,5 +80,5 @@ test/tmp:
 test/logs:
 	mkdir test/logs
 
-clean: pre-clean
+clean: pre-clean clean-db
 	rm -rfv test/tmp test/logs

README.md

@@ -1,51 +1,68 @@
-# GPG Lacre Project
+# Lacre Project
 
-Fork and continuation of original work of gpg-mailgate project: https://github.com/fkrone/gpg-mailgate
+**Lacre** (wax seal in Portuguese) is an add-on for Postfix that automatically
+encrypts incoming email before delivering it to recipients' inbox for
+recipients that have provided their public keys.
 
+Lacre is a fork and continuation of the original work on
+[gpg-mailgate](https://github.com/TheGreatGooo/gpg-mailgate) project.  It is
+still actively developed and should be considered as beta -- with all APIs and
+internals being subject to change.  Please only use this software if you know
+GnuPG well and accept occasional failures.
 
-**GPG Lacre** (wax seal in Portuguese) is a content filter for Postfix that automatically encrypts unencrypted incoming email using PGP or S/MIME for select recipients.
-This project is the continuation of the work of "gpg-mailgate" on providing open source, GnuPG based email encryption for emails at rest. All incoming emails are automatically encrypted with user's public key before they are saved on the server. It is a server side encryption solution while the control of the encryption keys are fully at the hands of the end-user and private keys are never stored on the server.
+# How it works
 
-The scope of the project is to improve on the already existing code, provide easy to use key upload system (standalone as well as Roundcube plugin) and key discoverability. Beside providing a solution that is easy to use we will also provide easy to digest material about encryption, how it works and how to make use of it in situations other the just mailbox encryption. Understanding how encryption works is the key to self-determination and is therefore an important part of the project.
+Lacre is a [content filter](https://www.postfix.org/FILTER_README.html).  This
+means, that when Postfix receives a message, it "forwards" that message to
+Lacre and if Lacre delivers it to a given destination, the message arrives to
+recipient's inbox.
 
-GPG Lacre will be battle tested on the email infrastructure of https://disroot.org (an ethical non-profit service provider).
+After receiving the message, Lacre does the following:
+
+1. If message already is encrypted, it just delivers the message immediately.
+2. Checks the list of recipients, finds their public keys if any were
+   provided.
+3. Encrypts message if possible.
+4. Delivers the message.
 
 ---
 
-The work on this project in 2021 is funded by https://nlnet.nl/thema/NGIZeroPET.html for which we are very thankful.
-
-The scope of the work for 2021 is:
- - Rewrite code to python3
- - Improve standalone key upload website
- - Provide Roundcube plugin for key management
- - Improve key server features
- - Provide webiste with information and tutorials on how to use GPG in general and also **Lacre**
- - (Optional) provide Autocrypt support
+Work on this project in 2021 was funded by
+[NGI Zero PET](https://nlnet.nl/thema/NGIZeroPET.html)
+for which we are very thankful.
 
 Made possible thanks to:<br>
 ![](https://nlnet.nl/logo/banner.png)
 
 ---
-For installation instructions, please refer to the included **INSTALL** file.
+
+# Installation
+
+For installation instructions, please refer to the included [INSTALL](INSTALL.md) file.
 
 ---
 
-# Features
+# Planned features
+
 - Correctly displays attachments and general email content; currently will only display first part of multipart messages
 - Public keys are stored in a dedicated gpg-home-directory
 - Encrypts both matching incoming and outgoing mail (this means gpg-mailgate can be used to encrypt outgoing mail for software that doesn't support PGP or S/MIME)
-- Decrypts PGP encrypted mails for present private keys (but no signature check and it does not always work with PGP/INLINE encrypted mails)
 - Easy installation
-- gpg-mailgate-web extension is a web interface allowing any user to upload PGP keys so that emails sent to them from your mail server will be encrypted (see gpg-mailgate-web directory for details)
-- people can submit their public key like to any keyserver to gpg-mailgate with the gpg-mailgate-web extension
-- people can send an S/MIME signed email to register@yourdomain.tld to register their public key
-- people can send their public OpenPGP key as attachment or inline to register@yourdomain.tld to register it
+- People can submit their public key like to any keyserver to gpg-mailgate with the gpg-mailgate-web extension
+- People can send an S/MIME signed email to register@yourdomain.tld to register their public key
+- People can send their public OpenPGP key as attachment or inline to register@yourdomain.tld to register it
+
+See also: [lacre-webgate](https://git.disroot.org/Lacre/lacre-webgate/) -- a
+web interface allowing any user to upload PGP keys so that emails sent to them
+from your mail server will be encrypted
 
 This is forked from the original project at http://code.google.com/p/gpg-mailgate/
 
 # Authors
 
-This is a combined work of many developers and contributors. We would like to pay honours to original gpg mailbox developers for making this project happen, and providing solid solution for encryption emails at rest:
+This is a combined work of many developers and contributors.  We would like to
+pay honours to original gpg mailbox developers for making this project happen,
+and providing solid solution for encryption emails at rest:
 
 * mcmaster <mcmaster@aphrodite.hurricanelabs.rsoc>
 * Igor Rzegocki <ajgon@irgon.com> - [GitHub](https://github.com/ajgon/gpg-mailgate)
@@ -57,4 +74,4 @@ This is a combined work of many developers and contributors. We would like to pa
 * Bruce Markey - [GitHub](https://github.com/TheEd1tor)
 * Remko Tronçon - [GitHub](https://github.com/remko/phkp/)
 * Kiritan Flux [GitHub](https://github.com/kflux)
-* Fabian Krone [GitHub] (https://github.com/fkrone/gpg-mailgate)
+* Fabian Krone [GitHub](https://github.com/fkrone/gpg-mailgate)

bin/lacreadm

@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# lacre
+#
+# This file is part of the lacre source code.
+#
+# lacre is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# lacre source code is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
+#
+
+python -m lacre.admin $*

doc/admin.md

@@ -0,0 +1,77 @@
+# Lacre administration
+
+## Command-line tool
+
+There's a little tool for administrators.  As long as Lacre Python packages
+are available via `PYTHONPATH`, you can use it like this:
+
+```sh
+python -m lacre.admin -h
+```
+
+Of course `-h` displays some help.
+
+**Note:** Help output includes information about the configuration file being
+in use, which may be useful at times.
+
+**Note:** You can also use a tiny shell wrapper around this tool, see
+`bin/lacreadm`.
+
+
+## Initialising database schema
+
+If you want to initialise Lacre's database (which is also used by the
+frontend), run:
+
+```sh
+python -m lacre.admin database -i
+```
+
+
+## Inspecting key confirmation queue
+
+To find out how many keys are waiting to be confirmed, run:
+
+```sh
+python -m lacre.admin queue
+```
+
+To see identities (emails) waiting for confirmation, use `--list` (or `-l`)
+option:
+
+```sh
+python -m lacre.admin queue -l
+```
+
+To delete one of these emails, use `--delete` (or `-d`) option:
+
+```sh
+python -m lacre.admin queue -d malory@example.org
+```
+
+## Inspecting identities registered
+
+To list all identities, run:
+
+```sh
+python -m lacre.admin identities -a
+```
+
+To preview a particular identity, run:
+
+```sh
+python -m lacre.admin identities -e alice@example.com
+```
+
+## Importing identities from existing GnuPG keyring
+
+If you already have a GnuPG keyring with your users' public keys or for some
+reason Lacre's identity database needs to be re-populated with identities,
+there's a command to do that:
+
+```sh
+python -m lacre.admin import -d /path/to/gnupg/directory
+```
+
+If you want to just re-populate the database, Lacre can remove all identities
+prior to importing keys -- just add `-r` flag.

doc/adv-filt.md

@@ -3,9 +3,9 @@
 ## Postfix Filters
 
 There are two types of Postfix mail filters: Simple Filters and Advanced
-Filters.  Simple Filters are executed for each incoming email as a new
-process, which may turn out to be expensive in terms of resources.  Advanced
-Filters work as a mail-processing proxies.
+Filters. Simple Filters are executed for each incoming email as a new
+process, which may turn out to be expensive in terms of resources. Advanced
+Filters work as a mail-processing proxies.
 
 For detailed documentation, see [FILTER README](https://www.postfix.org/FILTER_README.html).
 
@@ -13,29 +13,33 @@ For detailed documentation, see [FILTER README](https://www.postfix.org/FILTER_R
 
 Just use the following command to install dependencies:
 
-    pip install -r requirements.txt
+```
+pip install -r requirements.txt
+```
 
 ## Configuration
 
 Lacre Advanced Filter, also known as daemon, is configured in the `[daemon]`
-section of configuration file.  Two obligatory parameters to be defined there
+section of configuration file. Two obligatory parameters to be defined there
 are:
 
-  * `host` -- IP address or a host name;
+  * `host` -- IP address or a host name;
   * `port` -- TCP port Lacre should listen on.
 
 The other very important section is `[relay]`, which by default uses Simple
-Filter destination.  It has to be adjusted for Advanced Filter to work,
+Filter destination. It has to be adjusted for Advanced Filter to work,
 setting port to `10026`.
 
-Command to spawn a Lacre daemon process is:
+Command to spawn a Lacre daemon process is:
 
-    GPG_MAILGATE_CONFIG=/etc/gpg-mailgate.conf PYTHONPATH=... python -m lacre.daemon
+```
+LACRE_CONFIG=/etc/lacre.conf PYTHONPATH=... python -m lacre.daemon
+```
 
 Two environment variables used here are:
 
-  * `GPG_MAILGATE_CONFIG` (not mandatory) -- path to Lacre configuration,
-    unless it's kept in default location (`/etc/gpg-maillgate.conf`).
-  * `PYTHONPATH` (not mandatory) -- location of Lacre modules.  You can place
+  * `LACRE_CONFIG` (not mandatory) -- path to Lacre configuration,
+    unless it's kept in default location (`/etc/lacre.conf`).
+  * `PYTHONPATH` (not mandatory) -- location of Lacre modules. You can place
     them below your Python's `site-packages` to be reachable by any other
     Python software.

doc/testing.md

@@ -1,7 +1,7 @@
 # Testing
 
 First tests have been set up to cover GPG Mailgate with at least basic test
-that would be easy to run.  The tests are called "end-to-end", meaning that we
+that would be easy to run. The tests are called "end-to-end", meaning that we
 feed some input to GPG Mailgate and inspect the output.
 
 ## Running tests
@@ -10,42 +10,50 @@ To run tests, use command `make test`.
 
 There are 4 types of tests:
 
-  * `make e2etest` -- they cover a complete Lacre flow, from feeding it with
+  * `make e2etest` -- they cover a complete Lacre flow, from feeding it with
     an email to accepting its encrypted form;
   * `make daemontest` -- similar to the original `e2etest` goal, but tests the
     behaviour of the Lacre Daemon, i.e. key part of the Advanced Content
     Filter.
   * `make unittest` -- just small tests of small units of code;
-  * `make crontest` -- execute cron job with a SQLite database.
+  * `make crontest` -- execute cron job with a SQLite database.
 
 E2E tests (`make e2etest`) should produce some helpful logs, so inspect
 contents of `test/logs` directory if something goes wrong.
 
 If your system's Python binary isn't found in your `$PATH` or you want to use
-a specific binary, use make's macro overriding: `make test
+a specific binary, use make's macro overriding: `make test
 PYTHON=/path/to/python`.
 
 ## Key building blocks
 
 - *Test Script* (`test/e2e_test.py`) and *Daemon Test Script*
-  (`test/daemon_test.py`) that orchestrate the other components.  They perform
-  test cases described in the *Test Configuration*.  They spawn *Test Mail
+  (`test/daemon_test.py`) that orchestrate the other components. They perform
+  test cases described in the *Test Configuration*. They spawn *Test Mail
   Relay* and *GPG Mailgate* in appropriate order.
-- *Test Mail Relay* (`test/relay.py`), a simplistic mail daemon that only
-  supports the happy path.  It accepts a mail message and prints it to
+- *Test Mail Relay* (`test/relay.py`), a simplistic mail daemon that only
+  supports the happy path. It accepts a mail message and prints it to
   stdandard output.
 - *Test Configuration* (`test/e2e.ini`) specifies test cases: their input,
-  expected results and helpful documentation.  It also specifies the port that
+  expected results and helpful documentation. It also specifies the port that
   the *Test Mail Relay* should listen on.
 
 ## Limitations
 
 Currently tests only check if the message has been encrypted, without
-verifying that the correct key has been used.  That's because we don't know
-(yet) how to have a reproducible encrypted message.  Option
+verifying that the correct key has been used. That's because we don't know
+(yet) how to have a reproducible encrypted message. Option
 `--faked-system-time` wasn't enough to produce identical output.
 
 ## Troubleshooting
 
 When things go wrong, be sure to study `test/logs/e2e.log` and
 `test/logs/gpg-mailgate.log` files -- they contain some useful information.
+
+## Test identities
+
+There are several identities in test/keyhome and in the test database:
+
+* alice@disposlab: 1CD245308F0963D038E88357973CF4D9387C44D7
+* bob@disposlab: 19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67
+* evan@disposlab: 530B1BB2D0CC7971648198BBA4774E507D3AF5BC

gpg-mailgate.py

@@ -1,53 +0,0 @@
-#!/usr/bin/python
-#
-# gpg-mailgate
-#
-# This file is part of the gpg-mailgate source code.
-#
-# gpg-mailgate is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# gpg-mailgate source code is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with gpg-mailgate source code. If not, see <http://www.gnu.org/licenses/>.
-#
-
-import email
-import sys
-import time
-import logging
-
-import lacre
-import lacre.config as conf
-
-start = time.time()
-conf.load_config()
-lacre.init_logging(conf.get_item('logging', 'config'))
-
-# This has to be executed *after* logging initialisation.
-import lacre.mailgate as mailgate
-
-LOG = logging.getLogger(__name__)
-
-missing_params = conf.validate_config()
-if missing_params:
-    LOG.error(f"Aborting delivery! Following mandatory config parameters are missing: {missing_params!r}")
-    sys.exit(lacre.EX_CONFIG)
-
-# Read e-mail from stdin
-raw = sys.stdin.read()
-raw_message = email.message_from_string(raw)
-from_addr = raw_message['From']
-to_addrs = sys.argv[1:]
-
-# Let's start
-mailgate.deliver_message(raw_message, from_addr, to_addrs)
-(elapsed_s, process_t) = mailgate.exec_time_info(start)
-
-LOG.info("Elapsed-time: {elapsed:.2f}s; Process-time: {process:.4f}s".format(elapsed=elapsed_s, process=process_t))

lacre-logging.conf.sample

@@ -51,7 +51,7 @@ args=('/dev/log', handlers.SysLogHandler.LOG_MAIL)
 # Default Postfix log format.
 #
 [formatter_postfixfmt]
-format=%(asctime)s %(module)s[%(process)d]: %(message)s
+format=%(asctime)s %(name)s[%(process)d]: %(message)s
 datefmt=%b %e %H:%M:%S
 style=%
 validate=True

lacre.conf.sample

@@ -1,23 +1,13 @@
 [default]
-# Whether gpg-mailgate should add a header after it has processed an email
+# Whether lacre should add a header after it has processed an email
 # This may be useful for debugging purposes
 add_header = yes
 
 # Whether we should only encrypt emails if they are explicitly defined in
 # the key mappings below ([enc_keymap] section)
-# This means gpg-mailgate won't automatically detect PGP recipients for encrypting
+# This means lacre won't automatically detect PGP recipients for encrypting
 enc_keymap_only = no
 
-# Whether we should only decrypt emails if they are explicitly defined in
-# the key mappings below ([dec_keymap] section)
-# This means gpg-mailgate won't automatically detect PGP recipients for decrypting
-dec_keymap_only = no
-
-# If dec_keymap_only is set to yes and recipients have private keys present for decrypting
-# but are not on in the keymap, this can cause that mails for them will be
-# encrypted. Set this to no if you want this behaviour.
-failsave_dec = yes
-
 # Convert encrypted text/plain email to MIME-attached encrypt style.
 # (Default is to use older inline-style PGP encoding.)
 mime_conversion = yes
@@ -29,51 +19,37 @@ mime_conversion = yes
 # recipients all ignore the RFC you could this to yes.
 mail_case_insensitive = no
 
-# This setting disables PGP/INLINE decryption completely. However,
-# PGP/MIME encrypted mails will still be decrypted if possible. PGP/INLINE
-# decryption has to be seen as experimental and could have some negative
-# side effects. So if you want to take the risk set this to no.
-no_inline_dec = yes
-
-# Here you can define a regex for which the gateway should try to decrypt mails.
-# It could be used to define that decryption should be used for a wider range of
-# mail addresses e.g. a whole domain. No key is needed here. It is even active if 
-# dec_keymap is set to yes. If this feature should be disabled, don't leave it blank.
-# Set it to None. For further regex information please have a look at
-# https://docs.python.org/2/library/re.html
-dec_regex = None
-
 [gpg]
-# the directory where gpg-mailgate public keys are stored
+# the directory where lacre public keys are stored
 # (see INSTALL for details)
 #
 # Note that this directory should be accessible only for the Lacre user,
 # i.e. have mode 700.
-keyhome = /var/gpgmailgate/.gnupg
+keyhome = /var/lacre/.gnupg
 
 [smime]
 # the directory for the S/MIME certificate files
-cert_path = /var/gpgmailgate/smime
+cert_path = /var/lacre/smime
 
 [mailregister]
 # settings for the register-handler
 register_email = register@yourdomain.tld
-mail_templates = /var/gpgmailgate/register_templates
+mail_templates = /var/lacre/register_templates
 
 # URL to webpanel.  Upon receiving an email with a key, register-handler
 # uploads it to the web panel.
 webpanel_url = http://yourdomain.tld
 
 [cron]
-# settings for the gpgmw cron job
+# settings for the cron job
 send_email = yes
-notification_email = gpg-mailgate@yourdomain.tld
-mail_templates = /var/gpgmailgate/cron_templates
+notification_email = lacre@yourdomain.tld
+mail_templates = /var/lacre/cron_templates
 
 [logging]
 # path to the logging configuration; see documentation for details:
 # https://docs.python.org/3/library/logging.config.html#logging-config-fileformat
-config = /etc/gpg-lacre-logging.conf
+config = /etc/lacre-logging.conf
 
 [daemon]
 # Advanced Content Filter section.
@@ -83,14 +59,28 @@ config = /etc/gpg-lacre-logging.conf
 host = 127.0.0.1
 port = 10025
 
+# Maximum size (in bytes) of message body, i.e. data provided after DATA
+# message.  Following value comes from aiosmtpd module's default for this
+# setting.
+max_data_bytes = 33554432
+
+# Sometimes it may make sense to log additional information from mail headers.
+# This should never be PII, but information like encoding, content types, etc.
+log_headers = no
+
+# Sometimes we might fail to load keys and need to choose between delivering
+# in cleartext or not delivering.  The default is to deliver cleartext, but
+# administrators can make this decision on their own.
+bounce_on_keys_missing = no
+
 [relay]
 # the relay settings to use for Postfix
-# gpg-mailgate will submit email to this relay after it is done processing
+# lacre will submit email to this relay after it is done processing
 # unless you alter the default Postfix configuration, you won't have to modify this
 host = 127.0.0.1
 port = 10028
 # This is the default port of postfix. It is used to send some
-# mails through the GPG-Mailgate so they are encrypted
+# mails through the Lacre so they are encrypted
 enc_port = 25
 
 # Set this option to yes to use TLS for SMTP Servers which require TLS.
@@ -99,7 +89,7 @@ starttls = no
 [smtp]
 # Options when smtp auth is required to send out emails 
 enabled = false
-username = gpg-mailgate
+username = lacre
 password = changeme
 host = yourdomain.tld 
 port = 587
@@ -107,18 +97,42 @@ starttls = true
 
 [database]
 # edit the settings below if you want to read keys from a
-# gpg-mailgate-web database other than SQLite
+# lacre-webgate database other than SQLite
 enabled = yes
 url = sqlite:///test.db
 
-# For a MySQL database "gpgmw", user "gpgmw" and password "password",
+# Pooling mode: pessimistic or optimistic (required parameter).
+#
+# - Pessimistic disconnect-handling: pre_ping.  Connection pool will try using
+#   connection before it executes a SQL query to find out if the connection is
+#   still alive.  If not, it'll just establish a new connection.
+#
+# - Optimistic distonnect-handling: just avoid using connections after some
+#   time.
+#
+pooling_mode = optimistic
+
+# For a MySQL database "lacre", user "lacre" and password "password",
 # use the following URL:
 #
-#url = mysql://gpgmw:password@localhost/gpgmw
+#url = mysql://lacre:password@localhost/lacre
 #
 # For other RDBMS backends, see:
 # https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls
 
+# Number of seconds after which an idle connection is recycled.  This is
+# useful with MySQL servers.  This is only used with pooling_mode=optimistic.
+# For more information, see:
+# https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine.params.pool_recycle
+#max_connection_age = 3600
+
+# Number of connections stored in the pool.
+#pool_size = 5
+
+# If the pool size is not enough for current traffic, some connections can be
+# made and closed after use, to avoid pool growth and connection rejections.
+#max_overflow = 10
+
 [enc_keymap]
 # You can find these by running the following command:
 #	gpg --list-keys --keyid-format long user@example.com

lacre.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python
+#
+# lacre
+#
+# This file is part of the lacre source code.
+#
+# lacre is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# lacre source code is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import email
+from email.policy import SMTPUTF8
+import sys
+import logging
+
+import lacre
+import lacre.config as conf
+from lacre.stats import time_logger
+
+conf.load_config()
+lacre.init_logging(conf.get_item('logging', 'config'))
+
+# This has to be executed *after* logging initialisation.
+import lacre.core as core
+
+LOG = logging.getLogger('lacre.py')
+
+def main():
+    with time_logger('Message delivery', LOG):
+        missing_params = conf.validate_config()
+        config_file = conf.config_source()
+
+        if missing_params:
+            LOG.error(f"Aborting delivery! Following mandatory config parameters are missing in {config_file!r}: {missing_params}")
+            sys.exit(lacre.EX_CONFIG)
+
+        delivered = False
+        try:
+            # Read e-mail from stdin, parse it
+            raw = sys.stdin.read()
+            raw_message = email.message_from_string(raw, policy=SMTPUTF8)
+            from_addr = raw_message['From']
+            # Read recipients from the command-line
+            to_addrs = sys.argv[1:]
+
+            # Let's start
+            core.deliver_message(raw_message, from_addr, to_addrs)
+            delivered = True
+        except:
+            LOG.exception('Could not handle message')
+
+        if not delivered:
+            # It seems we weren't able to deliver the message.  In case it was
+            # some silly message-encoding issue that shouldn't bounce the
+            # message, we just try recoding the message body and delivering it.
+            try:
+                core.failover_delivery(raw_message, to_addrs, from_addr)
+            except:
+                LOG.exception('Failover delivery failed too')
+
+
+if __name__ == '__main__':
+    main()

lacre/__init__.py

@@ -8,30 +8,30 @@ import logging.config
 # be performed.  It only sets up a syslog handler, so that the admin has at
 # least some basic information.
 FAIL_OVER_LOGGING_CONFIG = {
-	'version': 1,
-	'formatters': {
-		'sysfmt': {
-			'format': '%(asctime)s %(module)s %(message)s',
-			'datefmt': '%Y-%m-%d %H:%M:%S'
-		},
-	},
-	'handlers': {
-		'syslog': {
-			'class': 'logging.handlers.SysLogHandler',
-			'level': 'INFO',
-			'formatter': 'sysfmt'
-		},
-		'lacrelog': {
-			'class': 'logging.FileHandler',
-			'level': 'INFO',
-			'formatter': 'sysfmt',
-			'filename': 'lacre.log'
-		}
-	},
-	'root': {
-		'level': 'INFO',
-		'handlers': ['syslog', 'lacrelog']
-	}
+    'version': 1,
+    'formatters': {
+        'sysfmt': {
+            'format': '%(asctime)s %(module)s %(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S'
+        },
+    },
+    'handlers': {
+        'syslog': {
+            'class': 'logging.handlers.SysLogHandler',
+            'level': 'INFO',
+            'formatter': 'sysfmt'
+        },
+        'lacrelog': {
+            'class': 'logging.FileHandler',
+            'level': 'INFO',
+            'formatter': 'sysfmt',
+            'filename': 'lacre.log'
+        }
+    },
+    'root': {
+        'level': 'INFO',
+        'handlers': ['syslog', 'lacrelog']
+    }
 }
 
 # Exit code taken from <sysexits.h>:
@@ -41,8 +41,9 @@ EX_CONFIG = 78
 
 
 def init_logging(config_filename):
-	if config_filename is not None:
-		logging.config.fileConfig(config_filename)
-	else:
-		logging.config.dictConfig(FAIL_OVER_LOGGING_CONFIG)
-		logging.warning('Lacre logging configuration missing, using syslog as default')
+    if config_filename is not None:
+        logging.config.fileConfig(config_filename)
+        logging.info('Configured from %s', config_filename)
+    else:
+        logging.config.dictConfig(FAIL_OVER_LOGGING_CONFIG)
+        logging.warning('Lacre logging configuration missing, using syslog as default')

lacre/_keyringcommon.py

@@ -0,0 +1,63 @@
+class KeyCache:
+    """A store for OpenPGP keys.
+
+    Key case is sanitised while loading from GnuPG if so
+    configured.  See mail_case_insensitive parameter in section
+    [default].
+    """
+
+    def __init__(self, keys: dict = None):
+        """Initialise an empty cache.
+
+        With keyring_dir given, set location of the directory from which keys should be loaded.
+        """
+        self._keys = keys
+
+    def __getitem__(self, fingerpring):
+        """Look up email assigned to the given fingerprint."""
+        return self._keys[fingerpring]
+
+    def __setitem__(self, fingerprint, email):
+        """Assign an email to a fingerpring, overwriting it if it was already present."""
+        self._keys[fingerprint] = email
+
+    def __contains__(self, fingerprint):
+        """Check if the given fingerprint is assigned to an email."""
+        # This method has to be present for KeyCache to be a dict substitute.
+        # See mailgate, function _identify_gpg_recipients.
+        return fingerprint in self._keys
+
+    def has_email(self, email):
+        """Check if cache contains a key assigned to the given email."""
+        return email in self._keys.values()
+
+    def __repr__(self):
+        """Return text representation of this object."""
+        details = ' '.join(self._keys.keys())
+        return '<KeyCache %s>' % (details)
+
+    def __iter__(self):
+        return iter(self._keys.keys())
+
+    def emails(self):
+        return { email: fingerprint for (fingerprint, email) in self._keys.items() }
+
+
+class KeyRing:
+    """Contract to be implemented by a key-store (a.k.a. keyring)."""
+
+    def freeze_identities(self) -> KeyCache:
+        """Return a static, async-safe copy of the identity map."""
+        raise NotImplementedError('KeyRing.load not implemented')
+
+    def register_or_update(self, email: str, key_id: str):
+        """Add a new (email,key) pair to the keystore."""
+        raise NotImplementedError('KeyRing.register_or_update not implemented')
+
+    def post_init_hook(self):
+        """Lets the keyring perform additional operations following its initialisation."""
+        pass
+
+    def shutdown(self):
+        """Lets the keyring perform operations prior to shutting down."""
+        pass

lacre/admin.py

@@ -0,0 +1,168 @@
+"""Lacre administrative tool.
+
+This is a command-line tool expected to be run by a person who knows what they
+are doing.  Also, please read the docs first.
+"""
+
+import sys
+import argparse
+import logging
+
+import GnuPG
+import lacre
+import lacre.config as conf
+
+conf.load_config()
+lacre.init_logging(conf.get_item('logging', 'config'))
+
+import lacre.repositories as repo
+import lacre.dbschema as db
+
+if __name__ == '__main__':
+    LOG = logging.getLogger('lacre.admin')
+else:
+    LOG = logging.getLogger(__name__)
+
+
+def _no_database():
+    print('Database unavailable or not configured properly')
+    sys.exit(lacre.EX_CONFIG)
+
+
+def sub_db(args):
+    """Sub-command to manipulate database."""
+    LOG.debug('Database operations ahead')
+
+    if args.init:
+        eng = repo.init_engine(conf.get_item('database', 'url'))
+        LOG.warning('Initialising database schema with engine: %s', eng)
+        print('Creating database tables')
+        db.create_tables(eng)
+
+
+def sub_queue(args):
+    """Sub-command to inspect queue contents."""
+    LOG.debug('Inspecting queue...')
+
+    eng = repo.init_engine(conf.get_item('database', 'url'))
+    queue = repo.KeyConfirmationQueue(engine=eng)
+
+    if args.delete:
+        queue.delete_key_by_email(args.delete)
+    elif args.list:
+        for k in queue.fetch_keys():
+            print(f'- {k.id}: {k.email}')
+    elif args.to_delete:
+        for k in queue.fetch_keys_to_delete():
+            print(f'- {k.id}: {k.email}')
+    else:
+        cnt = queue.count_keys()
+        if cnt is None:
+            _no_database()
+
+        print(f'Keys in the queue: {cnt}')
+
+
+def sub_identities(args):
+    """Sub-command to inspect identity database."""
+    LOG.debug('Inspecting identities...')
+
+    eng = repo.init_engine(conf.get_item('database', 'url'))
+    identities = repo.IdentityRepository(engine=eng)
+
+    all_identities = identities.freeze_identities()
+    if all_identities is None:
+        _no_database()
+
+    if args.email:
+        all_rev = all_identities.emails()
+        print('-', args.email, all_rev[args.email])
+    else:
+        for id_ in all_identities:
+            print('-', all_identities[id_], id_)
+
+
+def sub_import(args):
+    """Sub-command to import all identities known to GnuPG into Lacre database."""
+    LOG.debug('Importing identities...')
+    source_dir = args.homedir or conf.get_item('gpg', 'keyhome')
+
+    public = GnuPG.public_keys(source_dir)
+
+    eng = repo.init_engine(conf.get_item('database', 'url'))
+    identities = repo.IdentityRepository(engine=eng)
+
+    if args.reload:
+        identities.delete_all()
+
+    total = 0
+    for (fingerprint, email) in public.items():
+        LOG.debug('Importing %s - %s', email, fingerprint)
+        identities.register_or_update(email, fingerprint)
+        total += 1
+
+    LOG.debug('Imported %d identities', total)
+    print(f'Imported {total} identities')
+
+
+def main():
+    missing = conf.validate_config()
+    if missing:
+        LOG.error('Missing configuration parameters: %s', missing)
+        print('Insufficient configuration, aborting.')
+        sys.exit(lacre.EX_CONFIG)
+
+    general_conf = conf.config_source()
+    log_conf = conf.get_item('logging', 'config')
+
+    parser = argparse.ArgumentParser(
+        prog='lacre.admin',
+        description='Lacre Admin\'s best friend',
+        epilog=f'Config read from {general_conf}. For diagnostic info, see {log_conf}'
+        )
+
+    sub_commands = parser.add_subparsers(help='Sub-commands', required=True)
+
+    cmd_db = sub_commands.add_parser('database',
+        help='',
+        aliases=['db']
+        )
+    cmd_db.add_argument('-i', '--init', action='store_true',
+                        help='Initialise database schema')
+    cmd_db.set_defaults(operation=sub_db)
+
+    cmd_import = sub_commands.add_parser('import',
+        help='Load identities from GnuPG directory to Lacre database'
+        )
+    cmd_import.add_argument('-d', '--homedir', default=False,
+                            help='specify GnuPG directory (default: use configured dir.)')
+    cmd_import.add_argument('-r', '--reload', action='store_true',
+                            help='delete all keys from database before importing')
+    cmd_import.set_defaults(operation=sub_import)
+
+    cmd_queue = sub_commands.add_parser('queue',
+        help='Inspect key queue',
+        aliases=['q']
+        )
+    cmd_queue.add_argument('-D', '--delete',
+                           help='delete specified email from the queue')
+    cmd_queue.add_argument('-l', '--list', action='store_true',
+                           help='list keys in the queue')
+    cmd_queue.add_argument('-d', '--to-delete', action='store_true',
+                           help='list keys to be deleted')
+    cmd_queue.set_defaults(operation=sub_queue)
+
+    cmd_identities = sub_commands.add_parser('identities',
+        help='Inspect identity database',
+        aliases=['id']
+        )
+    cmd_identities.add_argument('-e', '--email', help='look up a single email')
+    cmd_identities.set_defaults(operation=sub_identities)
+
+    user_request = parser.parse_args()
+
+    user_request.operation(user_request)
+
+
+if __name__ == '__main__':
+    main()

lacre/config.py

@@ -1,24 +1,32 @@
-"""Lacre configuration
+"""Lacre configuration.
 
-Routines defined here are responsible for processing configuration.
+Routines defined here are responsible for processing and validating
+configuration.
 """
 
+from enum import Enum, auto
 from configparser import RawConfigParser
 
 import os
 
 
 # Environment variable name we read to retrieve configuration path.  This is to
-# enable non-root users to set up and run GPG Mailgate and to make the software
+# enable non-root users to set up and run Lacre and to make the software
 # testable.
-CONFIG_PATH_ENV = "GPG_MAILGATE_CONFIG"
+CONFIG_PATH_ENV = "LACRE_CONFIG"
 
 # List of mandatory configuration parameters.  Each item on this list should be
 # a pair: a section name and a parameter name.
 MANDATORY_CONFIG_ITEMS = [("relay", "host"),
                           ("relay", "port"),
                           ("daemon", "host"),
-                          ("daemon", "port")]
+                          ("daemon", "port"),
+                          ("gpg", "keyhome"),
+                          ('database', 'enabled'),
+                          ('database', 'url'),
+                          ('database', 'pooling_mode')]
+
+CRON_REQUIRED = [('cron', 'mail_templates')]
 
 # Global dict to keep configuration parameters.  It's hidden behind several
 # utility functions to make it easy to replace it with ConfigParser object in
@@ -32,17 +40,28 @@ def load_config() -> dict:
     If environment variable identified by CONFIG_PATH_ENV
     variable is set, its value is taken as a configuration file
     path.  Otherwise, the default is taken
-    ('/etc/gpg-mailgate.conf').
+    ('/etc/lacre.conf').
     """
-    configFile = os.getenv(CONFIG_PATH_ENV, '/etc/gpg-mailgate.conf')
+    config_file = config_source()
 
-    parser = _read_config(configFile)
+    parser = _read_config(config_file)
 
+    # XXX: Global variable.  It is a left-over from old GPG-Mailgate code.  We
+    # should drop it and probably use ConfigParser instance where configuration
+    # parameters are needed.
     global cfg
     cfg = _copy_to_dict(parser)
     return cfg
 
 
+def config_source() -> str:
+    """Return path of configuration file.
+
+    Taken from LACRE_CONFIG environment variable, and if it's not
+    set, defaults to /etc/lacre.conf."""
+    return os.getenv(CONFIG_PATH_ENV, '/etc/lacre.conf')
+
+
 def _read_config(fileName) -> RawConfigParser:
     cp = RawConfigParser()
     cp.read(fileName)
@@ -85,16 +104,23 @@ def flag_enabled(section, key) -> bool:
     return config_item_equals(section, key, 'yes')
 
 
-def validate_config():
+def validate_config(*, additional=None):
     """Check if configuration is complete.
 
     Returns a list of missing parameters, so an empty list means
     configuration is complete.
+
+    If 'additional' parameter is specified, it should be a list of
+    tuples (section, param).
     """
     missing = []
     for (section, param) in MANDATORY_CONFIG_ITEMS:
         if not config_item_set(section, param):
             missing.append((section, param))
+    if additional:
+        for (section, param) in additional:
+            if not config_item_set(section, param):
+                missing.append((section, param))
     return missing
 
 
@@ -115,3 +141,48 @@ def daemon_params():
 def strict_mode():
     """Check if Lacre is configured to support only a fixed list of keys."""
     return ("default" in cfg and cfg["default"]["enc_keymap_only"] == "yes")
+
+
+def should_log_headers() -> bool:
+    """Check if Lacre should log message headers."""
+    return flag_enabled('daemon', 'log_headers')
+
+
+class FromStrMixin:
+    """Additional operations for configuration enums."""
+
+    @classmethod
+    def from_str(cls, name, *, required=False):
+        if name is None:
+            return None
+
+        name = name.upper()
+
+        if name in cls.__members__:
+            return cls.__members__[name]
+
+        if required:
+            raise NameError('Unsupported or missing value')
+        else:
+            return None
+
+    @classmethod
+    def from_config(cls, section, key, *, required=False):
+        param = get_item(section, key)
+        return cls.from_str(param, required=required)
+
+
+class PGPStyle(FromStrMixin, Enum):
+    """PGP message structure: PGP/Inline or PGP/MIME."""
+    MIME = auto()
+    INLINE = auto()
+
+
+class PoolingMode(FromStrMixin, Enum):
+    """Database connection pool behaviour.
+
+    - Optimistic - recycles connections.
+    - Pessimistic - checks connection before usage.
+    """
+    OPTIMISTIC = auto()
+    PESSIMISTIC = auto()

lacre/core.py

@@ -0,0 +1,399 @@
+#
+# lacre
+#
+# This file is part of the lacre source code.
+#
+# lacre is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# lacre source code is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""Lacre's actual mail-delivery module.
+
+IMPORTANT: This module has to be loaded _after_ initialisation of the logging
+module.
+"""
+
+from email.mime.multipart import MIMEMultipart
+import copy
+import email
+from email.message import EmailMessage, MIMEPart
+import email.utils
+from email.policy import SMTPUTF8
+import GnuPG
+import asyncio
+from typing import Tuple
+
+import logging
+import lacre.text as text
+import lacre.config as conf
+import lacre.keyring as kcache
+import lacre.recipients as recpt
+import lacre.smime as smime
+from lacre.transport import send_msg, register_sender, SendFrom
+from lacre.mailop import KeepIntact, InlineOpenPGPEncrypt, MimeOpenPGPEncrypt, MailSerialisationException
+
+
+LOG = logging.getLogger(__name__)
+
+
+def _gpg_encrypt(raw_message, recipients):
+    if not conf.config_item_set('gpg', 'keyhome'):
+        LOG.error("No valid entry for gpg keyhome. Encryption aborted.")
+        return recipients
+
+    gpg_recipients, cleartext_recipients = \
+        recpt.identify_gpg_recipients(recipients, kcache.freeze_and_load_keys())
+
+    LOG.info(f"Got addresses: gpg_to={gpg_recipients!r}, ungpg_to={cleartext_recipients!r}")
+
+    if gpg_recipients:
+        LOG.info("Encrypting email to: %s", gpg_recipients)
+
+        mime, inline = _sort_gpg_recipients(gpg_recipients)
+
+        if mime:
+            # Encrypt mail with PGP/MIME
+            _gpg_encrypt_and_deliver(raw_message,
+                                     mime.keys(), mime.emails(),
+                                     _encrypt_all_payloads_mime)
+
+        if inline:
+            # Encrypt mail with PGP/INLINE
+            _gpg_encrypt_and_deliver(raw_message,
+                                     inline.keys(), inline.emails(),
+                                     _encrypt_all_payloads_inline)
+
+    LOG.info('Not processed emails: %s', cleartext_recipients)
+    return cleartext_recipients
+
+
+def _sort_gpg_recipients(gpg_to) -> Tuple[recpt.RecipientList, recpt.RecipientList]:
+    recipients_mime = list()
+    keys_mime = list()
+
+    recipients_inline = list()
+    keys_inline = list()
+
+    default_to_pgp_mime = conf.flag_enabled('default', 'mime_conversion')
+
+    for rcpt in gpg_to:
+        # Checking pre defined styles in settings first
+        style = conf.PGPStyle.from_config('pgp_style', rcpt.email())
+        if style is conf.PGPStyle.MIME:
+            recipients_mime.append(rcpt.email())
+            keys_mime.extend(rcpt.key().split(','))
+        elif style is conf.PGPStyle.INLINE:
+            recipients_inline.append(rcpt.email())
+            keys_inline.extend(rcpt.key().split(','))
+        else:
+            # Log message only if an unknown style is defined
+            if conf.config_item_set('pgp_style', rcpt.email()):
+                LOG.debug("Style %s for recipient %s is not known. Use default as fallback.",
+                          conf.get_item("pgp_style", rcpt.email()), rcpt.email())
+
+            # If no style is in settings defined for recipient, use default from settings
+            if default_to_pgp_mime:
+                recipients_mime.append(rcpt.email())
+                keys_mime.extend(rcpt.key().split(','))
+            else:
+                recipients_inline.append(rcpt.email())
+                keys_inline.extend(rcpt.key().split(','))
+
+    mime = recpt.RecipientList(recipients_mime, keys_mime)
+    inline = recpt.RecipientList(recipients_inline, keys_inline)
+
+    LOG.debug('Loaded recipients: MIME %s; Inline %s', repr(mime), repr(inline))
+
+    return mime, inline
+
+
+def _gpg_encrypt_copy(message: EmailMessage, keys, recipients, encrypt_f):
+    msg_copy = copy.deepcopy(message)
+    _customise_headers(msg_copy)
+    encrypted_payloads = encrypt_f(msg_copy, keys)
+    msg_copy.set_payload(encrypted_payloads)
+    return msg_copy
+
+
+def _gpg_encrypt_to_bytes(message: EmailMessage, keys, recipients, encrypt_f) -> bytes:
+    msg_copy = _gpg_encrypt_copy(message, keys, recipients, encrypt_f)
+    try:
+        return msg_copy.as_bytes(policy=SMTPUTF8)
+    except IndexError:
+        raise MailSerialisationException()
+
+
+def _gpg_encrypt_to_str(message: EmailMessage, keys, recipients, encrypt_f) -> str:
+    msg_copy = _gpg_encrypt_copy(message, keys, recipients, encrypt_f)
+    return msg_copy.as_string(policy=SMTPUTF8)
+
+
+def _gpg_encrypt_and_deliver(message: EmailMessage, keys, recipients, encrypt_f):
+    out = _gpg_encrypt_to_str(message, keys, recipients, encrypt_f)
+    send_msg(out, recipients)
+
+
+def _customise_headers(message: EmailMessage):
+    if conf.flag_enabled('default', 'add_header'):
+        message['X-Lacre'] = 'Encrypted by Lacre'
+
+
+def _encrypt_all_payloads_inline(message: EmailMessage, gpg_to_cmdline):
+
+    # This breaks cascaded MIME messages. Blame PGP/INLINE.
+    encrypted_payloads = list()
+    if isinstance(message.get_payload(), str):
+        return _encrypt_payload(message, gpg_to_cmdline).get_payload()
+
+    for payload in message.get_payload():
+        if(isinstance(payload.get_payload(), list)):
+            encrypted_payloads.extend(_encrypt_all_payloads_inline(payload, gpg_to_cmdline))
+        else:
+            encrypted_payloads.append(_encrypt_payload(payload, gpg_to_cmdline))
+
+    return encrypted_payloads
+
+
+def _encrypt_all_payloads_mime(message: EmailMessage, gpg_to_cmdline):
+    # Convert a plain text email into PGP/MIME attachment style.  Modeled after enigmail.
+    pgp_ver_part = MIMEPart()
+    pgp_ver_part.set_content('Version: 1' + text.EOL_S)
+    pgp_ver_part.set_type("application/pgp-encrypted")
+    pgp_ver_part.set_param('PGP/MIME version identification', "", 'Content-Description')
+
+    encrypted_part = MIMEPart()
+    encrypted_part.set_type("application/octet-stream")
+    encrypted_part.set_param('name', "encrypted.asc")
+    encrypted_part.set_param('OpenPGP encrypted message', "", 'Content-Description')
+    encrypted_part.set_param('inline', "",                'Content-Disposition')
+    encrypted_part.set_param('filename', "encrypted.asc", 'Content-Disposition')
+
+    message.preamble = "This is an OpenPGP/MIME encrypted message (RFC 2440 and 3156)"
+
+    boundary = _make_boundary()
+
+    if isinstance(message.get_payload(), str):
+        LOG.debug('Rewrapping a flat, text-only message')
+        wrapped_payload = _rewrap_payload(message)
+        encrypted_part.set_payload(wrapped_payload.as_string())
+
+        _set_type_and_boundary(message, boundary)
+
+        check_nested = True
+    else:
+        processed_payloads = _generate_message_from_payloads(message)
+        encrypted_part.set_payload(processed_payloads.as_string())
+
+        _set_type_and_boundary(message, boundary)
+
+        check_nested = False
+
+    return [pgp_ver_part, _encrypt_payload(encrypted_part, gpg_to_cmdline, check_nested)]
+
+
+def _rewrap_payload(message: EmailMessage) -> MIMEPart:
+    # In PGP/MIME (RFC 3156), the payload has to be a valid MIME entity.  In
+    # other words, we need to wrap text/* message's payload in a new MIME
+    # entity.
+
+    wrapper = MIMEPart(policy=SMTPUTF8)
+    content = message.get_content()
+    wrapper.set_content(content)
+
+    wrapper.set_type(message.get_content_type())
+
+    # Copy all Content-Type parameters.
+    for (pname, pvalue) in message.get_params(failobj=list()):
+        # Skip MIME type that's also returned by get_params().
+        if not '/' in pname:
+            wrapper.set_param(pname, pvalue)
+
+    return wrapper
+
+
+def _make_boundary():
+    junk_msg = MIMEMultipart()
+    # XXX See EmailTest.test_boundary_generated_after_as_string_call.
+    _ = junk_msg.as_string()
+    return junk_msg.get_boundary()
+
+
+def _set_type_and_boundary(message: EmailMessage, boundary):
+    message.set_type('multipart/encrypted')
+    message.set_param('protocol', 'application/pgp-encrypted')
+    message.set_param('boundary', boundary)
+
+
+def _encrypt_payload(payload: EmailMessage, recipients, check_nested=True, **kwargs):
+    raw_payload = payload.get_payload(decode=True)
+    LOG.debug('About to encrypt raw payload: %s', raw_payload)
+    LOG.debug('Original message: %s', payload)
+
+    if check_nested and text.is_payload_pgp_inline(raw_payload):
+        LOG.debug("Message is already pgp encrypted. No nested encryption needed.")
+        return payload
+
+    gpg = _make_encryptor(raw_payload, recipients)
+
+    gpg.update(raw_payload)
+    encrypted_data, exit_code = gpg.encrypt()
+
+    payload.set_payload(encrypted_data)
+    isAttachment = payload.get_param('attachment', None, 'Content-Disposition') is not None
+
+    if isAttachment:
+        _append_gpg_extension(payload)
+
+    return payload
+
+
+def _make_encryptor(raw_data, recipients):
+    # No check is needed for conf.get_item('gpg', 'keyhome') as this is already
+    # done in method gpg_encrypt
+    keyhome = conf.get_item('gpg', 'keyhome')
+
+    if isinstance(raw_data, str):
+        return GnuPG.GPGEncryptor(keyhome, recipients, 'utf-8')
+    else:
+        return GnuPG.GPGEncryptor(keyhome, recipients)
+
+
+def _append_gpg_extension(attachment):
+    filename = attachment.get_filename()
+    if not filename:
+        return
+
+    pgpFilename = filename + ".pgp"
+
+    # Attachment name can come from one of two places: Content-Disposition or
+    # Content-Type header, hence the two cases below.
+
+    if not (attachment.get('Content-Disposition') is None):
+        attachment.set_param('filename', pgpFilename, 'Content-Disposition')
+
+    if not (attachment.get('Content-Type') is None) and not (attachment.get_param('name') is None):
+        attachment.set_param('name', pgpFilename)
+
+
+def _generate_message_from_payloads(payloads, message=None):
+    if message is None:
+        message = email.mime.multipart.MIMEMultipart(payloads.get_content_subtype())
+
+    for payload in payloads.get_payload():
+        if(isinstance(payload.get_payload(), list)):
+            message.attach(_generate_message_from_payloads(payload))
+        else:
+            message.attach(payload)
+
+    return message
+
+
+def _get_first_payload(payloads):
+    if payloads.is_multipart():
+        return _get_first_payload(payloads.get_payload(0))
+    else:
+        return payloads
+
+
+def _recode(m: EmailMessage):
+    payload = m.get_payload()
+    m.set_content(payload)
+
+
+def failover_delivery(message: EmailMessage, recipients, from_address):
+    """Try delivering message just one last time."""
+    LOG.debug('Failover delivery')
+
+    send = SendFrom(from_address)
+    if message.get_content_maintype() == 'text':
+        LOG.debug('Flat text message, adjusting coding')
+        _recode(message)
+        b = message.as_bytes(policy=SMTPUTF8)
+        send(b, recipients)
+    elif message.get_content_maintype() == 'multipart':
+        LOG.debug('Multipart message, adjusting coding of text entities')
+        for part in message.iter_parts():
+            if part.get_content_maintype() == 'text':
+                _recode(part)
+        b = message.as_bytes(policy=SMTPUTF8)
+        send(b, recipients)
+    else:
+        LOG.warning('No failover strategy, giving up')
+
+
+def _is_encrypted(raw_message: EmailMessage):
+    if raw_message.get_content_type() == 'multipart/encrypted':
+        return True
+
+    first_part = _get_first_payload(raw_message)
+    if first_part.get_content_type() == 'application/pkcs7-mime':
+        return True
+
+    return text.is_message_pgp_inline(first_part)
+
+
+def delivery_plan(recipients, message: EmailMessage, key_cache: kcache.KeyCache):
+    """Generate a sequence of delivery strategies."""
+    if _is_encrypted(message):
+        LOG.debug('Message is already encrypted: %s', message)
+        return [KeepIntact(recipients)]
+
+    gpg_recipients, cleartext_recipients = recpt.identify_gpg_recipients(recipients, key_cache)
+
+    mime, inline = _sort_gpg_recipients(gpg_recipients)
+
+    keyhome = conf.get_item('gpg', 'keyhome')
+
+    plan = []
+    if mime:
+        plan.append(MimeOpenPGPEncrypt(mime.emails(), mime.keys(), keyhome))
+    if inline:
+        plan.append(InlineOpenPGPEncrypt(inline.emails(), inline.keys(), keyhome))
+    if cleartext_recipients:
+        plan.append(KeepIntact(cleartext_recipients))
+
+    return plan
+
+
+def deliver_message(raw_message: EmailMessage, from_address, to_addrs):
+    """Send RAW_MESSAGE to all TO_ADDRS using the best encryption method available."""
+    # Ugly workaround to keep the code working without too many changes.
+    register_sender(from_address)
+
+    sanitize = text.choose_sanitizer(conf.get_item('default', 'mail_case_insensitive'))
+    recipients_left = [sanitize(recipient) for recipient in to_addrs]
+
+    send = SendFrom(from_address)
+
+    # There is no need for nested encryption
+    LOG.debug("Seeing if it's already encrypted")
+    if _is_encrypted(raw_message):
+        LOG.debug("Message is already encrypted. Encryption aborted.")
+        send(raw_message.as_string(), recipients_left)
+        return
+
+    # Encrypt mails for recipients with known public PGP keys
+    LOG.debug("Encrypting with OpenPGP")
+    recipients_left = _gpg_encrypt(raw_message, recipients_left)
+    if not recipients_left:
+        return
+
+    # Encrypt mails for recipients with known S/MIME certificate
+    LOG.debug("Encrypting with S/MIME")
+    recipients_left = smime.encrypt(raw_message, recipients_left, from_address)
+    if not recipients_left:
+        return
+
+    # Send out mail to recipients which are left
+    LOG.debug("Sending the rest as text/plain")
+    send(raw_message.as_bytes(policy=SMTPUTF8), recipients_left)

lacre/daemon.py

@@ -2,30 +2,27 @@
 
 import logging
 import lacre
+from lacre.text import DOUBLE_EOL_BYTES
+from lacre.stats import time_logger
 import lacre.config as conf
 import sys
 from aiosmtpd.controller import Controller
 from aiosmtpd.smtp import Envelope
 import asyncio
 import email
-import time
-from watchdog.observers import Observer
-
-# Mail status constants.
-#
-# These are the only values that our mail handler is allowed to return.
-RESULT_OK = '250 OK'
-RESULT_ERROR = '500 Could not process your message'
-RESULT_NOT_IMPLEMENTED = '500 Not implemented yet'
+from email.policy import SMTPUTF8
 
 # Load configuration and init logging, in this order.  Only then can we load
-# the last Lacre module, i.e. lacre.mailgate.
+# the last Lacre module, i.e. lacre.core.
 conf.load_config()
 lacre.init_logging(conf.get_item("logging", "config"))
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger('lacre.daemon')
 
-import lacre.mailgate as gate
+from GnuPG import EncryptionException
+import lacre.core as gate
 import lacre.keyring as kcache
+import lacre.transport as xport
+from lacre.mailop import KeepIntact, MailSerialisationException
 
 
 class MailEncryptionProxy:
@@ -37,35 +34,63 @@ class MailEncryptionProxy:
 
     async def handle_DATA(self, server, session, envelope: Envelope):
         """Accept a message and either encrypt it or forward as-is."""
-        start = time.process_time()
-        try:
-            keys = await self._keyring.freeze_identities()
-            message = email.message_from_bytes(envelope.content)
-            for operation in gate.delivery_plan(envelope.rcpt_tos, keys):
-                LOG.debug(f"Sending mail via {operation!r}")
-                new_message = operation.perform(message)
-                gate.send_msg(new_message, operation.recipients(), envelope.mail_from)
-        except TypeError as te:
-            LOG.exception("Got exception while processing", exc_info=te)
-            return RESULT_ERROR
+        with time_logger('Message delivery', LOG):
+            try:
+                keys = self._keyring.freeze_identities()
+                message = email.message_from_bytes(envelope.original_content, policy=SMTPUTF8)
 
-        ellapsed = time.process_time() - start
-        LOG.info(f'Message delivered in {ellapsed} ms')
-        return RESULT_OK
+                if message.defects:
+                    LOG.warning("Issues found: %d; %s", len(message.defects), repr(message.defects))
+
+                send = xport.SendFrom(envelope.mail_from)
+                for operation in gate.delivery_plan(envelope.rcpt_tos, message, keys):
+                    LOG.debug(f"Sending mail via {operation!r}")
+                    try:
+                        new_message = operation.perform(message)
+                        send(new_message, operation.recipients())
+                    except (EncryptionException, MailSerialisationException, UnicodeEncodeError):
+                        # If the message can't be encrypted, deliver cleartext.
+                        LOG.exception('Unable to encrypt message, delivering in cleartext')
+                        if not isinstance(operation, KeepIntact):
+                            self._send_unencrypted(operation, envelope, send)
+                        else:
+                            LOG.exception('Cannot perform: %s', operation)
+                            raise
+
+            except:
+                LOG.exception('Unexpected exception caught, bouncing message')
+
+                if conf.should_log_headers():
+                    LOG.error('Erroneous message headers: %s', self._beginning(envelope))
+
+                return xport.RESULT_ERRORR
+
+        return xport.RESULT_OK
+
+    def _send_unencrypted(self, operation, envelope, send: xport.SendFrom):
+        # Do not parse and re-generate the message, just send it as it is.
+        send(envelope.original_content, operation.recipients())
+
+    def _beginning(self, e: Envelope) -> bytes:
+        double_eol_pos = e.original_content.find(DOUBLE_EOL_BYTES)
+        if double_eol_pos < 0:
+            limit = len(e.original_content)
+        else:
+            limit = double_eol_pos
+        end = min(limit, 2560)
+        return e.original_content[0:end]
+
+    def _seconds_between(self, start_ms, end_ms) -> float:
+        return (end_ms - start_ms) * 1000
 
 
-def _init_controller(keys: kcache.KeyRing, tout: float = 5):
+def _init_controller(keys: kcache.KeyRing, max_body_bytes=None, tout: float = 5):
     proxy = MailEncryptionProxy(keys)
     host, port = conf.daemon_params()
     LOG.info(f"Initialising a mail Controller at {host}:{port}")
-    return Controller(proxy, hostname=host, port=port, ready_timeout=tout)
-
-
-def _init_reloader(keyring_dir: str, reloader) -> kcache.KeyringModificationListener:
-    listener = kcache.KeyringModificationListener(reloader)
-    observer = Observer()
-    observer.schedule(listener, keyring_dir, recursive=False)
-    return observer
+    return Controller(proxy, hostname=host, port=port,
+                      ready_timeout=tout,
+                      data_size_limit=max_body_bytes)
 
 
 def _validate_config():
@@ -85,32 +110,38 @@ async def _sleep():
         await asyncio.sleep(360)
 
 
-def _main():
+async def _main():
     _validate_config()
 
     keyring_path = conf.get_item('gpg', 'keyhome')
-    keyring = kcache.KeyRing(keyring_path)
-    controller = _init_controller(keyring)
-    reloader = _init_reloader(keyring_path, keyring)
+    max_data_bytes = int(conf.get_item('daemon', 'max_data_bytes', 2**25))
 
-    LOG.info(f'Watching keyring directory {keyring_path}...')
-    reloader.start()
-
-    LOG.info('Starting the daemon...')
-    controller.start()
+    loop = asyncio.get_event_loop()
 
     try:
-        asyncio.run(_sleep())
+        keyring = kcache.init_keyring()
+        controller = _init_controller(keyring, max_data_bytes)
+
+        keyring.post_init_hook()
+
+        LOG.info('Starting the daemon with GnuPG=%s, socket=%s, database=%s',
+                 keyring_path,
+                 conf.daemon_params(),
+                 conf.get_item('database', 'url'))
+        controller.start()
+
+        await _sleep()
     except KeyboardInterrupt:
         LOG.info("Finishing...")
+    except:
+        LOG.exception('Unexpected exception caught, your system may be unstable')
     finally:
         LOG.info('Shutting down keyring watcher and the daemon...')
-        reloader.stop()
-        reloader.join()
+        keyring.shutdown()
         controller.stop()
 
     LOG.info("Done")
 
 
 if __name__ == '__main__':
-    _main()
+    asyncio.run(_main())

lacre/dbschema.py

@@ -0,0 +1,64 @@
+"""Database schema for Lacre.
+
+This definition includes:
+
+- 'lacre_keys' -- temporary key storage, used by the frontend to submit keys and
+  by webgate-cron script to import submitted keys.
+
+- 'lacre_identities' -- identity catalogue, used by encryption logic to match
+  emails with corresponding keys.
+
+- 'lacre_locks' -- used only by the frontend.
+"""
+
+import sqlalchemy
+
+# Values for lacre_keys.status column:
+# - ST_DEFAULT: initial state;
+# - ST_IMPORTED: key has been successfully processed by cron job;
+# - ST_TO_BE_DELETED: key can be deleted.
+ST_DEFAULT, ST_IMPORTED, ST_TO_BE_DELETED = range(3)
+
+# lacre_keys.confirmed is set to an empty string when a key is confirmed by the user.
+CO_CONFIRMED = ''
+
+_meta = sqlalchemy.MetaData()
+
+LACRE_KEYS = sqlalchemy.Table('lacre_keys', _meta,
+                              sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, nullable=False, autoincrement='auto'),
+                              sqlalchemy.Column('email', sqlalchemy.String(256), index=True),
+                              # ASCII-armored key
+                              sqlalchemy.Column('publickey', sqlalchemy.Text),
+                              # Empty string means this key has been confirmed.
+                              sqlalchemy.Column('confirm', sqlalchemy.String(32)),
+                              # Status: see ST_* constants at the top of the file.
+                              sqlalchemy.Column('status', sqlalchemy.Integer, nullable=False, default=0),
+                              sqlalchemy.Column('time', sqlalchemy.DateTime))
+
+LACRE_LOCKS = sqlalchemy.Table('lacre_locks', _meta,
+                               sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, nullable=False, autoincrement='auto'),
+                               sqlalchemy.Column('ip', sqlalchemy.String(16)),
+                               sqlalchemy.Column('time', sqlalchemy.Integer),
+                               sqlalchemy.Column('action', sqlalchemy.String(16)),
+                               sqlalchemy.Column('num', sqlalchemy.Integer),
+                               )
+
+LACRE_IDENTITIES = sqlalchemy.Table('lacre_identities', _meta,
+                                    sqlalchemy.Column('email', sqlalchemy.String(256), index=True, nullable=False),
+                                    # Key fingerprint
+                                    sqlalchemy.Column('fingerprint', sqlalchemy.String(64), index=True, nullable=False))
+
+def init_identities_table() -> sqlalchemy.Table:
+    return LACRE_IDENTITIES
+
+def init_locks_table() -> sqlalchemy.Table:
+    return LACRE_LOCKS
+
+def init_keys_table() -> sqlalchemy.Table:
+    return LACRE_KEYS
+
+def create_tables(engine):
+    _meta.create_all(engine)
+
+def table_metadata():
+    return _meta

lacre/keyring.py

@@ -4,144 +4,25 @@ IMPORTANT: This module has to be loaded _after_ initialisation of the logging
 module.
 """
 
-import lacre.text as text
+import lacre.config as conf
+from lacre._keyringcommon import KeyRing, KeyCache
+from lacre.repositories import IdentityRepository, init_engine
 import logging
-from os import stat
-from watchdog.events import FileSystemEventHandler
-from asyncio import Semaphore, run
-import copy
-
-import GnuPG
 
 LOG = logging.getLogger(__name__)
 
 
-def _sanitize(keys):
-    for fingerprint in keys:
-        keys[fingerprint] = text.sanitize_case_sense(keys[fingerprint])
-
-    return keys
+def init_keyring() -> KeyRing:
+    """Initialise appropriate type of keyring."""
+    url = conf.get_item('database', 'url')
+    db_engine = init_engine(url)
+    return IdentityRepository(engine=db_engine)
 
 
-class KeyCacheMisconfiguration(Exception):
-    """Exception used to signal that KeyCache is misconfigured."""
+def freeze_and_load_keys() -> KeyCache:
+    """Load and return keys.
 
-
-class KeyCache:
-    """A store for OpenPGP keys.
-
-    Key case is sanitised while loading from GnuPG if so
-    configured.  See mail_case_insensitive parameter in section
-    [default].
+    Doesn't refresh the keys when they change on disk.
     """
-
-    def __init__(self, keys: dict = None):
-        """Initialise an empty cache.
-
-        With keyring_dir given, set location of the directory from which keys should be loaded.
-        """
-        self._keys = keys
-
-    def __getitem__(self, fingerpring):
-        """Look up email assigned to the given fingerprint."""
-        return self._keys[fingerpring]
-
-    def __setitem__(self, fingerprint, email):
-        """Assign an email to a fingerpring, overwriting it if it was already present."""
-        self._keys[fingerprint] = email
-
-    def __contains__(self, fingerprint):
-        """Check if the given fingerprint is assigned to an email."""
-        # This method has to be present for KeyCache to be a dict substitute.
-        # See mailgate, function _identify_gpg_recipients.
-        return fingerprint in self._keys
-
-    def has_email(self, email):
-        """Check if cache contains a key assigned to the given email."""
-        return email in self._keys.values()
-
-
-class KeyRing:
-    """A high-level adapter for GnuPG-maintained keyring directory.
-
-    Its role is to keep a cache of keys present in the keyring,
-    reload it when necessary and produce static copies of
-    fingerprint=>email maps.
-    """
-
-    def __init__(self, path: str):
-        """Initialise the adapter."""
-        self._path = path
-        self._keys = self._load_and_sanitize()
-        self._sema = Semaphore()
-        self._last_mod = None
-
-    def _load_and_sanitize(self):
-        keys = self._load_keyring_from(self._path)
-        return _sanitize(keys)
-
-    def _load_keyring_from(self, keyring_dir):
-        return GnuPG.public_keys(keyring_dir)
-
-    async def freeze_identities(self) -> KeyCache:
-        """Return a static, async-safe copy of the identity map."""
-        async with self._sema:
-            keys = copy.deepcopy(self._keys)
-            return KeyCache(keys)
-
-    def load(self):
-        """Load keyring, replacing any previous contents of the cache."""
-        LOG.debug('Reloading keys...')
-        run(self._load())
-
-    async def _load(self):
-        last_mod = self._read_mod_time()
-        if self._is_modified(last_mod):
-            async with self._sema:
-                self.replace_keyring(self._load_keyring_from(self._path))
-
-        self._last_mod = self._read_mod_time()
-
-    reload = load
-
-    def replace_keyring(self, keys: dict):
-        """Overwrite previously stored key cache with KEYS."""
-        keys = _sanitize(keys)
-
-        LOG.info(f'Storing {len(keys)} keys')
-        self._keys = keys
-
-    def _read_mod_time(self):
-        # (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)
-        MTIME = 8
-        st = stat(self._path)
-        return st[MTIME]
-
-    def _is_modified(self, last_mod):
-        if self._last_mod is None:
-            LOG.debug('Keyring not loaded before')
-            return True
-        elif self._last_mod != last_mod:
-            LOG.debug('Keyring directory mtime changed')
-            return True
-        else:
-            LOG.debug('Keyring not modified ')
-            return False
-
-
-class KeyringModificationListener(FileSystemEventHandler):
-    """A filesystem event listener that triggers key cache reload."""
-
-    def __init__(self, keyring: KeyRing):
-        """Initialise a listener with a callback to be executed upon each change."""
-        self._keyring = keyring
-
-    def handle(self, event):
-        """Reload keys upon FS event."""
-        LOG.debug(f'Reloading on event {event!r}')
-        self._keyring.reload()
-
-    # All methods should do the same: reload the key cache.
-    # on_created = handle
-    # on_deleted = handle
-    on_modified = handle
+    keyring = init_keyring()
+    return keyring.freeze_identities()

lacre/mailgate.py

@@ -1,567 +0,0 @@
-"""Lacre's actual mail-delivery module.
-
-IMPORTANT: This module has to be loaded _after_ initialisation of the logging
-module.
-"""
-
-#
-# gpg-mailgate
-#
-# This file is part of the gpg-mailgate source code.
-#
-# gpg-mailgate is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# gpg-mailgate source code is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with gpg-mailgate source code. If not, see <http://www.gnu.org/licenses/>.
-#
-
-from email.mime.multipart import MIMEMultipart
-import copy
-import email
-import email.message
-import email.utils
-import GnuPG
-import os
-import smtplib
-import sys
-import time
-import asyncio
-
-# imports for S/MIME
-from M2Crypto import BIO, SMIME, X509
-
-import logging
-import lacre.text as text
-import lacre.config as conf
-import lacre.keyring as kcache
-from lacre.mailop import KeepIntact, InlineOpenPGPEncrypt, MimeOpenPGPEncrypt
-
-
-LOG = logging.getLogger(__name__)
-
-
-def _gpg_encrypt(raw_message, recipients):
-    if not conf.config_item_set('gpg', 'keyhome'):
-        LOG.error("No valid entry for gpg keyhome. Encryption aborted.")
-        return recipients
-
-    gpg_to, ungpg_to = _identify_gpg_recipients(recipients, _load_keys())
-
-    LOG.info(f"Got addresses: gpg_to={gpg_to!r}, ungpg_to={ungpg_to!r}")
-
-    if gpg_to:
-        LOG.info("Encrypting email to: %s" % ' '.join(x.email() for x in gpg_to))
-
-        gpg_to_smtp_mime, gpg_to_cmdline_mime, \
-            gpg_to_smtp_inline, gpg_to_cmdline_inline = \
-            _sort_gpg_recipients(gpg_to)
-
-        if gpg_to_smtp_mime:
-            # Encrypt mail with PGP/MIME
-            _gpg_encrypt_and_deliver(raw_message,
-                                     gpg_to_cmdline_mime, gpg_to_smtp_mime,
-                                     _encrypt_all_payloads_mime)
-
-        if gpg_to_smtp_inline:
-            # Encrypt mail with PGP/INLINE
-            _gpg_encrypt_and_deliver(raw_message,
-                                     gpg_to_cmdline_inline, gpg_to_smtp_inline,
-                                     _encrypt_all_payloads_inline)
-
-    LOG.info(f"Not processed emails: {ungpg_to}")
-    return ungpg_to
-
-
-def _sort_gpg_recipients(gpg_to):
-    gpg_to_smtp_mime = list()
-    gpg_to_cmdline_mime = list()
-
-    gpg_to_smtp_inline = list()
-    gpg_to_cmdline_inline = list()
-
-    default_to_pgp_mime = conf.config_item_equals('default', 'mime_conversion', 'yes')
-
-    for rcpt in gpg_to:
-        # Checking pre defined styles in settings first
-        if conf.config_item_equals('pgp_style', rcpt.email(), 'mime'):
-            gpg_to_smtp_mime.append(rcpt.email())
-            gpg_to_cmdline_mime.extend(rcpt.key().split(','))
-        elif conf.config_item_equals('pgp_style', rcpt.email(), 'inline'):
-            gpg_to_smtp_inline.append(rcpt.email())
-            gpg_to_cmdline_inline.extend(rcpt.key().split(','))
-        else:
-            # Log message only if an unknown style is defined
-            if conf.config_item_set('pgp_style', rcpt.email()):
-                LOG.debug("Style %s for recipient %s is not known. Use default as fallback."
-                          % (conf.get_item("pgp_style", rcpt.email()), rcpt.email()))
-
-            # If no style is in settings defined for recipient, use default from settings
-            if default_to_pgp_mime:
-                gpg_to_smtp_mime.append(rcpt.email())
-                gpg_to_cmdline_mime.extend(rcpt.key().split(','))
-            else:
-                gpg_to_smtp_inline.append(rcpt.email())
-                gpg_to_cmdline_inline.extend(rcpt.key().split(','))
-
-    return gpg_to_smtp_mime, gpg_to_cmdline_mime, gpg_to_smtp_inline, gpg_to_cmdline_inline
-
-
-def _gpg_encrypt_and_return(message, cmdline, to, encrypt_f) -> str:
-    msg_copy = copy.deepcopy(message)
-    _customise_headers(msg_copy)
-    encrypted_payloads = encrypt_f(msg_copy, cmdline)
-    msg_copy.set_payload(encrypted_payloads)
-    return msg_copy.as_string()
-
-
-def _gpg_encrypt_and_deliver(message, cmdline, to, encrypt_f):
-    out = _gpg_encrypt_and_return(message, cmdline, to, encrypt_f)
-    send_msg(out, to)
-
-
-def _customise_headers(msg_copy):
-    if conf.config_item_equals('default', 'add_header', 'yes'):
-        msg_copy['X-GPG-Mailgate'] = 'Encrypted by GPG Mailgate'
-
-    if 'Content-Transfer-Encoding' in msg_copy:
-        msg_copy.replace_header('Content-Transfer-Encoding', '8BIT')
-    else:
-        msg_copy['Content-Transfer-Encoding'] = '8BIT'
-
-
-def _load_keys():
-    """Return a map from a key's fingerprint to email address."""
-    keyring = kcache.KeyRing(conf.get_item('gpg', 'keyhome'))
-    return asyncio.run(keyring.freeze_identities())
-
-
-class GpgRecipient:
-    """A tuple-like object that contains GPG recipient data."""
-
-    def __init__(self, left, right):
-        """Initialise a tuple-like object that contains GPG recipient data."""
-        self._left = left
-        self._right = right
-
-    def __getitem__(self, index):
-        """Pretend this object is a tuple by returning an indexed tuple element."""
-        if index == 0:
-            return self._left
-        elif index == 1:
-            return self._right
-        else:
-            raise IndexError()
-
-    def __repr__(self):
-        """Return textual representation of this GPG Recipient."""
-        return f"GpgRecipient({self._left!r}, {self._right!r})"
-
-    def email(self):
-        """Return this recipient's email address."""
-        return self._left
-
-    def key(self):
-        """Return this recipient's key ID."""
-        return self._right
-
-
-def _identify_gpg_recipients(recipients, keys: kcache.KeyCache):
-    # This list will be filled with pairs (M, N), where M is the destination
-    # address we're going to deliver the message to and N is the identity we're
-    # going to encrypt it for.
-    gpg_to = list()
-
-    # This will be the list of recipients that haven't provided us with their
-    # public keys.
-    ungpg_to = list()
-
-    # In "strict mode", only keys included in configuration are used to encrypt
-    # email.
-    strict_mode = conf.strict_mode()
-
-    # GnuPG keys found in our keyring.
-
-    LOG.info(f'Processisng recipients: {recipients!r}; keys: {keys!r}')
-    for to in recipients:
-        LOG.info(f"At to={to!r}")
-        own_key = _try_configured_key(to, keys)
-        if own_key is not None:
-            gpg_to.append(GpgRecipient(own_key[0], own_key[1]))
-            continue
-
-        direct_key = _try_direct_key_lookup(to, keys, strict_mode)
-        if direct_key is not None:
-            gpg_to.append(GpgRecipient(direct_key[0], direct_key[1]))
-            continue
-
-        domain_key = _try_configured_domain_key(to, keys)
-        if domain_key is not None:
-            gpg_to.append(GpgRecipient(domain_key[0], domain_key[1]))
-            continue
-
-        ungpg_to.append(to)
-
-    LOG.debug(f'Collected recipients;  GPG: {gpg_to};  UnGPG: {ungpg_to}')
-    return gpg_to, ungpg_to
-
-
-def _find_key(recipient, keys, strict_mode):
-    own_key = _try_configured_key(recipient, keys)
-    if own_key is not None:
-        return own_key
-
-    direct_key = _try_direct_key_lookup(recipient, keys, strict_mode)
-    if direct_key is not None:
-        return direct_key
-
-    domain_key = _try_configured_domain_key(recipient, keys)
-    if domain_key is not None:
-        return domain_key
-
-    return None
-
-
-def _try_configured_key(recipient, keys):
-    if conf.config_item_set('enc_keymap', recipient):
-        key = conf.get_item('enc_keymap', recipient)
-        if key in keys:
-            LOG.debug(f"Found key {key} configured for {recipient}")
-            return (recipient, key)
-
-    LOG.debug(f"No configured key found for {recipient}")
-    return None
-
-
-def _try_direct_key_lookup(recipient, keys, strict_mode):
-    if strict_mode:
-        return None
-
-    if keys.has_email(recipient):
-        LOG.info(f"Found key for {recipient}")
-        return recipient, recipient
-
-    (newto, topic) = text.parse_delimiter(recipient)
-    if keys.has_email(newto):
-        LOG.info(f"Found key for {newto}, stripped {recipient}")
-        return recipient, newto
-
-    return None
-
-
-def _try_configured_domain_key(recipient, keys):
-    parts = recipient.split('@')
-    if len(parts) != 2:
-        return None
-
-    domain = parts[1]
-    if conf.config_item_set('enc_domain_keymap', domain):
-        domain_key = conf.get_item('enc_domain_keymap', domain)
-        if domain_key in keys:
-            LOG.debug(f"Found domain key {domain_key} for {recipient}")
-            return recipient, domain_key
-
-    LOG.debug(f"No domain key for {recipient}")
-    return None
-
-
-def _encrypt_all_payloads_inline(message, gpg_to_cmdline):
-
-    # This breaks cascaded MIME messages. Blame PGP/INLINE.
-    encrypted_payloads = list()
-    if isinstance(message.get_payload(), str):
-        return _encrypt_payload(message, gpg_to_cmdline).get_payload()
-
-    for payload in message.get_payload():
-        if(isinstance(payload.get_payload(), list)):
-            encrypted_payloads.extend(_encrypt_all_payloads_inline(payload, gpg_to_cmdline))
-        else:
-            encrypted_payloads.append(_encrypt_payload(payload, gpg_to_cmdline))
-
-    return encrypted_payloads
-
-
-def _encrypt_all_payloads_mime(message, gpg_to_cmdline):
-    # Convert a plain text email into PGP/MIME attachment style.  Modeled after enigmail.
-    pgp_ver_part = email.message.Message()
-    pgp_ver_part.set_payload("Version: 1"+text.EOL)
-    pgp_ver_part.set_type("application/pgp-encrypted")
-    pgp_ver_part.set_param('PGP/MIME version identification', "", 'Content-Description')
-
-    encrypted_part = email.message.Message()
-    encrypted_part.set_type("application/octet-stream")
-    encrypted_part.set_param('name', "encrypted.asc")
-    encrypted_part.set_param('OpenPGP encrypted message', "", 'Content-Description')
-    encrypted_part.set_param('inline', "",                'Content-Disposition')
-    encrypted_part.set_param('filename', "encrypted.asc", 'Content-Disposition')
-
-    if isinstance(message.get_payload(), str):
-        # WTF!  It seems to swallow the first line.  Not sure why.  Perhaps
-        # it's skipping an imaginary blank line someplace. (ie skipping a header)
-        # Workaround it here by prepending a blank line.
-        # This happens only on text only messages.
-        additionalSubHeader = ""
-        encoding = sys.getdefaultencoding()
-        if 'Content-Type' in message and not message['Content-Type'].startswith('multipart'):
-            additionalSubHeader = "Content-Type: " + message['Content-Type'] + text.EOL
-            (base, encoding) = text.parse_content_type(message['Content-Type'])
-            LOG.debug(f"Identified encoding as {encoding}")
-        encrypted_part.set_payload(additionalSubHeader+text.EOL + message.get_payload(decode=True).decode(encoding))
-        check_nested = True
-    else:
-        processed_payloads = _generate_message_from_payloads(message)
-        encrypted_part.set_payload(processed_payloads.as_string())
-        check_nested = False
-
-    message.preamble = "This is an OpenPGP/MIME encrypted message (RFC 2440 and 3156)"
-
-    # Use this just to generate a MIME boundary string.
-    junk_msg = MIMEMultipart()
-    junk_str = junk_msg.as_string()  # WTF!  Without this, get_boundary() will return 'None'!
-    boundary = junk_msg.get_boundary()
-
-    # This also modifies the boundary in the body of the message, ie it gets parsed.
-    if 'Content-Type' in message:
-        message.replace_header('Content-Type', f"multipart/encrypted; protocol=\"application/pgp-encrypted\"; boundary=\"{boundary}\""+text.EOL)
-    else:
-        message['Content-Type'] = f"multipart/encrypted; protocol=\"application/pgp-encrypted\"; boundary=\"{boundary}\""+text.EOL
-
-    return [pgp_ver_part, _encrypt_payload(encrypted_part, gpg_to_cmdline, check_nested)]
-
-
-def _encrypt_payload(payload, gpg_to_cmdline, check_nested=True):
-    raw_payload = payload.get_payload(decode=True)
-    if check_nested and text.is_pgp_inline(raw_payload):
-        LOG.debug("Message is already pgp encrypted. No nested encryption needed.")
-        return payload
-
-    # No check is needed for conf.get_item('gpg', 'keyhome') as this is already
-    # done in method gpg_encrypt
-    gpg = GnuPG.GPGEncryptor(conf.get_item('gpg', 'keyhome'), gpg_to_cmdline,
-                             payload.get_content_charset())
-    gpg.update(raw_payload)
-    encrypted_data, returncode = gpg.encrypt()
-    LOG.debug("Return code from encryption=%d (0 indicates success)." % returncode)
-    if returncode != 0:
-        LOG.info("Encrytion failed with return code %d. Encryption aborted." % returncode)
-        return payload
-
-    payload.set_payload(encrypted_data)
-    isAttachment = payload.get_param('attachment', None, 'Content-Disposition') is not None
-
-    if isAttachment:
-        filename = payload.get_filename()
-        if filename:
-            pgpFilename = filename + ".pgp"
-            if not (payload.get('Content-Disposition') is None):
-                payload.set_param('filename', pgpFilename, 'Content-Disposition')
-            if not (payload.get('Content-Type') is None) and not (payload.get_param('name') is None):
-                payload.set_param('name', pgpFilename)
-    if not (payload.get('Content-Transfer-Encoding') is None):
-        payload.replace_header('Content-Transfer-Encoding', "7bit")
-
-    return payload
-
-
-def _smime_encrypt(raw_message, recipients):
-    global LOG
-    global from_addr
-
-    if not conf.config_item_set('smime', 'cert_path'):
-        LOG.info("No valid path for S/MIME certs found in config file. S/MIME encryption aborted.")
-        return recipients
-
-    cert_path = conf.get_item('smime', 'cert_path')+"/"
-    s = SMIME.SMIME()
-    sk = X509.X509_Stack()
-    smime_to = list()
-    unsmime_to = list()
-
-    for addr in recipients:
-        cert_and_email = _get_cert_for_email(addr[0], cert_path)
-
-        if not (cert_and_email is None):
-            (to_cert, normal_email) = cert_and_email
-            LOG.debug("Found cert " + to_cert + " for " + addr + ": " + normal_email)
-            smime_to.append(addr)
-            x509 = X509.load_cert(to_cert, format=X509.FORMAT_PEM)
-            sk.push(x509)
-        else:
-            unsmime_to.append(addr)
-
-    if smime_to:
-        s.set_x509_stack(sk)
-        s.set_cipher(SMIME.Cipher('aes_192_cbc'))
-        p7 = s.encrypt(BIO.MemoryBuffer(raw_message.as_string()))
-        # Output p7 in mail-friendly format.
-        out = BIO.MemoryBuffer()
-        out.write('From: ' + from_addr + text.EOL)
-        out.write('To: ' + raw_message['To'] + text.EOL)
-        if raw_message['Cc']:
-            out.write('Cc: ' + raw_message['Cc'] + text.EOL)
-        if raw_message['Bcc']:
-            out.write('Bcc: ' + raw_message['Bcc'] + text.EOL)
-        if raw_message['Subject']:
-            out.write('Subject: ' + raw_message['Subject'] + text.EOL)
-
-        if conf.config_item_equals('default', 'add_header', 'yes'):
-            out.write('X-GPG-Mailgate: Encrypted by GPG Mailgate' + text.EOL)
-
-        s.write(out, p7)
-
-        LOG.debug(f"Sending message from {from_addr} to {smime_to}")
-
-        send_msg(out.read(), smime_to)
-    if unsmime_to:
-        LOG.debug(f"Unable to find valid S/MIME certificates for {unsmime_to}")
-
-    return unsmime_to
-
-
-def _get_cert_for_email(to_addr, cert_path):
-    insensitive = conf.config_item_equals('default', 'mail_case_insensitive', 'yes')
-
-    LOG.info(f'Retrieving certificate for {to_addr!r} from {cert_path!r}, sensitivity={insensitive!r}')
-
-    files_in_directory = os.listdir(cert_path)
-    for filename in files_in_directory:
-        file_path = os.path.join(cert_path, filename)
-        if not os.path.isfile(file_path):
-            continue
-
-        if insensitive:
-            if filename.casefold() == to_addr:
-                return (file_path, to_addr)
-        else:
-            if filename == to_addr:
-                return (file_path, to_addr)
-
-    # support foo+ignore@bar.com -> foo@bar.com
-    LOG.info(f"An email with topic? {to_addr}")
-    (fixed_up_email, topic) = text.parse_delimiter(to_addr)
-    LOG.info(f'Got {fixed_up_email!r} and {topic!r}')
-    if topic is None:
-        # delimiter not used
-        LOG.info('Topic not found')
-        return None
-    else:
-        LOG.info(f"Looking up certificate for {fixed_up_email} after parsing {to_addr}")
-        return _get_cert_for_email(fixed_up_email, cert_path)
-
-
-def _generate_message_from_payloads(payloads, message=None):
-    if message is None:
-        message = email.mime.multipart.MIMEMultipart(payloads.get_content_subtype())
-
-    for payload in payloads.get_payload():
-        if(isinstance(payload.get_payload(), list)):
-            message.attach(_generate_message_from_payloads(payload))
-        else:
-            message.attach(payload)
-
-    return message
-
-
-def _get_first_payload(payloads):
-    if payloads.is_multipart():
-        return _get_first_payload(payloads.get_payload(0))
-    else:
-        return payloads
-
-
-def send_msg(message: str, recipients, fromaddr=None):
-    """Send MESSAGE to RECIPIENTS to the mail relay."""
-    global from_addr
-
-    if fromaddr is not None:
-        from_addr = fromaddr
-
-    recipients = [_f for _f in recipients if _f]
-    if recipients:
-        LOG.info(f"Sending email to: {recipients!r}")
-        relay = conf.relay_params()
-        smtp = smtplib.SMTP(relay[0], relay[1])
-        if conf.flag_enabled('relay', 'starttls'):
-            smtp.starttls()
-        smtp.sendmail(from_addr, recipients, message)
-    else:
-        LOG.info("No recipient found")
-
-
-def _is_encrypted(raw_message):
-    if raw_message.get_content_type() == 'multipart/encrypted':
-        return True
-
-    first_part = _get_first_payload(raw_message)
-    if first_part.get_content_type() == 'application/pkcs7-mime':
-        return True
-
-    first_payload = first_part.get_payload(decode=True)
-    return text.is_pgp_inline(first_payload)
-
-
-def delivery_plan(recipients, key_cache: kcache.KeyCache):
-    """Generate a sequence of delivery strategies."""
-    gpg_to, ungpg_to = _identify_gpg_recipients(recipients, key_cache)
-
-    gpg_mime_to, gpg_mime_cmd, gpg_inline_to, gpg_inline_cmd = \
-        _sort_gpg_recipients(gpg_to)
-
-    keyhome = conf.get_item('gpg', 'keyhome')
-
-    plan = []
-    if gpg_mime_to:
-        plan.append(MimeOpenPGPEncrypt(gpg_mime_to, gpg_mime_cmd, keyhome))
-    if gpg_inline_to:
-        plan.append(InlineOpenPGPEncrypt(gpg_inline_to, gpg_inline_cmd, keyhome))
-    if ungpg_to:
-        plan.append(KeepIntact(ungpg_to))
-
-    return plan
-
-
-def deliver_message(raw_message: email.message.Message, from_address, to_addrs):
-    """Send RAW_MESSAGE to all TO_ADDRS using the best encryption method available."""
-    global from_addr
-
-    # Ugly workaround to keep the code working without too many changes.
-    from_addr = from_address
-
-    recipients_left = [text.sanitize_case_sense(recipient) for recipient in to_addrs]
-
-    # There is no need for nested encryption
-    LOG.debug("Seeing if it's already encrypted")
-    if _is_encrypted(raw_message):
-        LOG.debug("Message is already encrypted. Encryption aborted.")
-        send_msg(raw_message.as_string(), recipients_left)
-        return
-
-    # Encrypt mails for recipients with known public PGP keys
-    LOG.debug("Encrypting with OpenPGP")
-    recipients_left = _gpg_encrypt(raw_message, recipients_left)
-    if not recipients_left:
-        return
-
-    # Encrypt mails for recipients with known S/MIME certificate
-    LOG.debug("Encrypting with S/MIME")
-    recipients_left = _smime_encrypt(raw_message, recipients_left)
-    if not recipients_left:
-        return
-
-    # Send out mail to recipients which are left
-    LOG.debug("Sending the rest as text/plain")
-    send_msg(raw_message.as_string(), recipients_left)
-
-
-def exec_time_info(start_timestamp):
-    """Calculate time since the given timestamp."""
-    elapsed_s = time.time() - start_timestamp
-    process_t = time.process_time()
-    return (elapsed_s, process_t)

lacre/mailop.py

@@ -14,13 +14,19 @@ There are 3 operations available:
 """
 
 import logging
-import lacre.mailgate as mailgate
+import lacre.core as core
 from email.message import Message
+from email.policy import SMTP, SMTPUTF8
 
 
 LOG = logging.getLogger(__name__)
 
 
+class MailSerialisationException(BaseException):
+    """We can't turn an EmailMessage into sequence of bytes."""
+    pass
+
+
 class MailOperation:
     """Contract for an operation to be performed on a message."""
 
@@ -28,7 +34,7 @@ class MailOperation:
         """Initialise the operation with a recipient."""
         self._recipients = recipients
 
-    def perform(self, message: Message):
+    def perform(self, message: Message) -> bytes:
         """Perform this operation on MESSAGE.
 
         Return target message.
@@ -69,12 +75,12 @@ class InlineOpenPGPEncrypt(OpenPGPEncrypt):
         """Initialise strategy object."""
         super().__init__(recipients, keys, keyhome)
 
-    def perform(self, msg: Message):
+    def perform(self, msg: Message) -> bytes:
         """Encrypt with PGP Inline."""
         LOG.debug('Sending PGP/Inline...')
-        return mailgate._gpg_encrypt_and_return(msg,
-                                                self._keys, self._recipients,
-                                                mailgate._encrypt_all_payloads_inline)
+        return core._gpg_encrypt_to_bytes(msg,
+                                          self._keys, self._recipients,
+                                          core._encrypt_all_payloads_inline)
 
 
 class MimeOpenPGPEncrypt(OpenPGPEncrypt):
@@ -84,12 +90,12 @@ class MimeOpenPGPEncrypt(OpenPGPEncrypt):
         """Initialise strategy object."""
         super().__init__(recipients, keys, keyhome)
 
-    def perform(self, msg: Message):
+    def perform(self, msg: Message) -> bytes:
         """Encrypt with PGP MIME."""
         LOG.debug('Sending PGP/MIME...')
-        return mailgate._gpg_encrypt_and_return(msg,
-                                                self._keys, self._recipients,
-                                                mailgate._encrypt_all_payloads_mime)
+        return core._gpg_encrypt_to_bytes(msg,
+                                          self._keys, self._recipients,
+                                          core._encrypt_all_payloads_mime)
 
 
 class SMimeEncrypt(MailOperation):
@@ -101,10 +107,10 @@ class SMimeEncrypt(MailOperation):
         self._email = email
         self._cert = certificate
 
-    def perform(self, message: Message):
+    def perform(self, message: Message) -> bytes:
         """Encrypt with a certificate."""
         LOG.warning(f"Delivering clear-text to {self._recipients}")
-        return message
+        return message.as_bytes(policy=SMTP)
 
     def __repr__(self):
         """Generate a representation with just method and key."""
@@ -121,9 +127,12 @@ class KeepIntact(MailOperation):
         """Initialise pass-through operation for a given recipient."""
         super().__init__(recipients)
 
-    def perform(self, message: Message):
+    def perform(self, message: Message) -> bytes:
         """Return MESSAGE unmodified."""
-        return message.as_string()
+        try:
+            return message.as_bytes(policy=SMTPUTF8)
+        except IndexError as e:
+            raise MailSerialisationException(e)
 
     def __repr__(self):
         """Return representation with just method and email."""

lacre/notify.py

@@ -0,0 +1,54 @@
+"""Lacre notification sender"""
+
+import logging
+import lacre
+import lacre.config as conf
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+import markdown
+
+
+LOG = logging.getLogger(__name__)
+
+
+def _load_file(name):
+    f = open(name)
+    data = f.read()
+    f.close()
+    return data
+
+
+def _authenticate_maybe(smtp):
+    if conf.config_item_equals('smtp', 'enabled', 'true'):
+        LOG.debug(f"Connecting to {conf.get_item('smtp', 'host')}:{conf.get_item('smtp', 'port')}")
+        smtp.connect(conf.get_item('smtp', 'host'), conf.get_item('smtp', 'port'))
+        smtp.ehlo()
+        if conf.config_item_equals('smtp', 'starttls', 'true'):
+            LOG.debug("StartTLS enabled")
+            smtp.starttls()
+        smtp.ehlo()
+        smtp.login(conf.get_item('smtp', 'username'), conf.get_item('smtp', 'password'))
+
+
+def notify(mailsubject, messagefile, recipients = None):
+    """Send notification email."""
+
+    mailbody = _load_file(conf.get_item('cron', 'mail_templates') + "/" + messagefile)
+    msg = MIMEMultipart("alternative")
+
+    msg["From"] = conf.get_item('cron', 'notification_email')
+    msg["To"] = recipients
+    msg["Subject"] = mailsubject
+
+    msg.attach(MIMEText(mailbody, 'plain'))
+    msg.attach(MIMEText(markdown.markdown(mailbody), 'html'))
+
+    if conf.config_item_set('relay', 'host') and conf.config_item_set('relay', 'enc_port'):
+        (host, port) = conf.relay_params()
+        smtp = smtplib.SMTP(host, port)
+        _authenticate_maybe(smtp)
+        LOG.info('Delivering notification: %s', recipients)
+        smtp.sendmail(conf.get_item('cron', 'notification_email'), recipients, msg.as_string())
+    else:
+        LOG.warning("Could not send mail due to wrong configuration")

lacre/recipients.py

@@ -0,0 +1,202 @@
+#
+# lacre
+#
+# This file is part of the lacre source code.
+#
+# lacre is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# lacre source code is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""Recipient processing package.
+
+Defines:
+- GpgRecipient, wrapper for user's email and identity.
+- RecipientList, a wrapper for lists of GpgRecipient objects.
+"""
+
+import logging
+import lacre.config as conf
+import lacre.keyring as kcache
+import lacre.text as text
+
+LOG = logging.getLogger(__name__)
+
+
+class Recipient:
+    """Wraps recipient's email."""
+
+    def __init__(self, email):
+        """Initialise the recipient."""
+        self._email = email
+
+    def email(self) -> str:
+        """Return email address of this recipient."""
+        return self._email
+
+    def __str__(self):
+        """Return string representation of this recipient: the email address."""
+        return self._email
+
+
+class GpgRecipient(Recipient):
+    """A tuple-like object that contains GPG recipient data."""
+
+    def __init__(self, left, right):
+        """Initialise a tuple-like object that contains GPG recipient data."""
+        super().__init__(left)
+        self._right = right
+
+    def __getitem__(self, index):
+        """Pretend this object is a tuple by returning an indexed tuple element."""
+        if index == 0:
+            return self.email()
+        elif index == 1:
+            return self._right
+        else:
+            raise IndexError()
+
+    def __repr__(self):
+        """Return textual representation of this GPG Recipient."""
+        return f"GpgRecipient({self.email()!r}, {self._right!r})"
+
+    __str__ = __repr__
+
+    def key(self):
+        """Return this recipient's key ID."""
+        return self._right
+
+
+class RecipientList:
+    """Encalsulates two lists of recipients.
+
+    First list contains addresses, the second - GPG identities.
+    """
+
+    def __init__(self, recipients=[], keys=[]):
+        """Initialise lists of recipients and identities."""
+        self._recipients = [GpgRecipient(email, key) for (email, key) in zip(recipients, keys)]
+
+    def emails(self):
+        """Return list of recipients."""
+        return [r.email() for r in self._recipients]
+
+    def keys(self):
+        """Return list of GPG identities."""
+        return [r.key() for r in self._recipients]
+
+    def __iadd__(self, recipient: GpgRecipient):
+        """Append a recipient."""
+        LOG.debug('Adding %s to %s', recipient, self._recipients)
+        self._recipients.append(recipient)
+        LOG.debug('Added; got: %s', self._recipients)
+        return self
+
+    def __len__(self):
+        """Provide len().
+
+        With this method, it is possible to write code like:
+
+        rl = RecipientList()
+        if rl:
+            # do something
+        """
+        return len(self._recipients)
+
+    def __repr__(self):
+        """Returns textual object representation."""
+        return '<RecipientList %d %s>' % (len(self._recipients), ','.join(self.emails()))
+
+
+def identify_gpg_recipients(recipients, keys: kcache.KeyCache):
+    """Split recipient list into GPG and non-GPG ones."""
+    # This list will be filled with pairs (M, N), where M is the destination
+    # address we're going to deliver the message to and N is the identity we're
+    # going to encrypt it for.
+    gpg_recipients = list()
+
+    # This will be the list of recipients that haven't provided us with their
+    # public keys.
+    cleartext_recipients = list()
+
+    # In "strict mode", only keys included in configuration are used to encrypt
+    # email.
+    strict_mode = conf.strict_mode()
+
+    for recipient in recipients:
+        gpg_recipient = _find_key(recipient, keys, strict_mode)
+        if gpg_recipient is not None:
+            gpg_recipients.append(gpg_recipient)
+        else:
+            cleartext_recipients.append(recipient)
+
+    LOG.debug('Collected recipients; GPG: %s; cleartext: %s', gpg_recipients, cleartext_recipients)
+    return gpg_recipients, cleartext_recipients
+
+
+def _find_key(recipient, keys: kcache.KeyCache, strict_mode):
+    own_key = _try_configured_key(recipient, keys)
+    if own_key is not None:
+        return GpgRecipient(own_key[0], own_key[1])
+
+    direct_key = _try_direct_key_lookup(recipient, keys, strict_mode)
+    if direct_key is not None:
+        return GpgRecipient(direct_key[0], direct_key[1])
+
+    domain_key = _try_configured_domain_key(recipient, keys)
+    if domain_key is not None:
+        return GpgRecipient(domain_key[0], domain_key[1])
+
+    return None
+
+
+def _try_configured_key(recipient, keys: kcache.KeyCache):
+    if conf.config_item_set('enc_keymap', recipient):
+        key = conf.get_item('enc_keymap', recipient)
+        if key in keys:
+            LOG.debug(f"Found key {key} configured for {recipient}")
+            return (recipient, key)
+
+    LOG.debug(f"No configured key found for {recipient}")
+    return None
+
+
+def _try_direct_key_lookup(recipient, keys: kcache.KeyCache, strict_mode):
+    if strict_mode:
+        return None
+
+    if keys.has_email(recipient):
+        LOG.info(f"Found key for {recipient}")
+        return recipient, recipient
+
+    (newto, topic) = text.parse_delimiter(recipient)
+    if keys.has_email(newto):
+        LOG.info(f"Found key for {newto}, stripped {recipient}")
+        return recipient, newto
+
+    return None
+
+
+def _try_configured_domain_key(recipient, keys: kcache.KeyCache):
+    parts = recipient.split('@')
+    if len(parts) != 2:
+        return None
+
+    domain = parts[1]
+    if conf.config_item_set('enc_domain_keymap', domain):
+        domain_key = conf.get_item('enc_domain_keymap', domain)
+        if domain_key in keys:
+            LOG.debug(f"Found domain key {domain_key} for {recipient}")
+            return recipient, domain_key
+
+    LOG.debug(f"No domain key for {recipient}")
+    return None

lacre/repositories.py

@@ -0,0 +1,200 @@
+"""Lacre identity and key repositories."""
+
+from sqlalchemy import create_engine, select, delete, and_, func
+from sqlalchemy.exc import OperationalError
+import logging
+
+from lacre.config import flag_enabled, config_item_set, get_item, PoolingMode
+from lacre._keyringcommon import KeyRing, KeyCache
+import lacre.dbschema as db
+
+LOG = logging.getLogger(__name__)
+
+
+_HOUR_IN_SECONDS = 3600
+
+# Internal state
+_engine = None
+
+
+def init_engine(url, db_debug=False):
+    global _engine
+
+    if not _engine:
+        config = _conn_config(db_debug)
+        _engine = create_engine(url, **config)
+
+    return _engine
+
+
+def _conn_config(db_debug):
+    config = dict()
+
+    mode = PoolingMode.from_config('database', 'pooling_mode', required=True)
+    if mode is PoolingMode.OPTIMISTIC:
+        # Optimistic distonnect-handling: recycle connections.
+        config['pool_recycle'] = int(get_item('database', 'max_connection_age', _HOUR_IN_SECONDS))
+    elif mode is PoolingMode.PESSIMISTIC:
+        # Pessimistic disconnect-handling: pre_ping.
+        config['pool_pre_ping'] = True
+
+    # Additional pool settings
+    if config_item_set('database', 'pool_size'):
+        config['pool_size'] = int(get_item('database', 'pool_size'))
+
+    if config_item_set('database', 'max_overflow'):
+        config['max_overflow'] = int(get_item('database', 'max_overflow'))
+
+    if db_debug:
+        config['echo'] = 'debug'
+        config['echo_pool'] = 'debug'
+
+    LOG.debug('Database engine configuration: %s', config)
+    return config
+
+
+class IdentityRepository(KeyRing):
+    def __init__(self, /, connection=None, *, engine):
+        self._identities = db.LACRE_IDENTITIES
+        self._engine = engine
+
+    def register_or_update(self, email, fprint):
+        assert email, "email is mandatory"
+        assert fprint, "fprint is mandatory"
+
+        if self._exists(email):
+            self._update(email, fprint)
+        else:
+            self._insert(email, fprint)
+
+    def _exists(self, email: str) -> bool:
+        selq = select(self._identities.c.email).where(self._identities.c.email == email)
+        with self._engine.connect() as conn:
+            return [e for e in conn.execute(selq)]
+
+    def _insert(self, email, fprint):
+        insq = self._identities.insert().values(email=email, fingerprint=fprint)
+
+        LOG.debug('Registering identity: %s -- %s', insq, insq.compile().params)
+        with self._engine.connect() as conn:
+            conn.execute(insq)
+
+    def _update(self, email, fprint):
+        upq = self._identities.update() \
+            .values(fingerprint=fprint) \
+            .where(self._identities.c.email == email)
+
+        LOG.debug('Updating identity: %s -- %s', upq, upq.compile().params)
+        with self._engine.connect() as conn:
+            conn.execute(upq)
+
+    def delete(self, email):
+        delq = delete(self._identities).where(self._identities.c.email == email)
+        LOG.debug('Deleting assigned keys: %s -- %s', delq, delq.compile().params)
+
+        with self._engine.connect() as conn:
+            conn.execute(delq)
+
+    def delete_all(self):
+        LOG.warn('Deleting all identities from the database')
+
+        delq = delete(self._identities)
+        with self._engine.connect() as conn:
+            conn.execute(delq)
+
+    def freeze_identities(self) -> KeyCache:
+        """Return a static, async-safe copy of the identity map.
+
+        Depending on the value of [daemon]bounce_on_keys_missing value,
+        if we get a database exception, this method will either return
+        empty collection or let the exception be propagated.
+        """
+        try:
+            return self._load_identities()
+        except OperationalError:
+            if flag_enabled('daemon', 'bounce_on_keys_missing'):
+                raise
+            else:
+                LOG.exception('Failed to load keys, returning empty collection')
+                return KeyCache({})
+
+    def _load_identities(self) -> KeyCache:
+        all_identities = select(self._identities.c.fingerprint, self._identities.c.email)
+        with self._engine.connect() as conn:
+            result = conn.execute(all_identities)
+            LOG.debug('Retrieving all keys: %s', all_identities)
+            return KeyCache({key_id: email for key_id, email in result})
+
+
+class KeyConfirmationQueue:
+    """Encapsulates access to lacre_keys table."""
+
+    # Default number of items retrieved from the database.
+    keys_read_max = 100
+
+    def __init__(self, /, engine):
+        self._keys = db.LACRE_KEYS
+        self._engine = engine
+
+    def fetch_keys(self, /, max_keys=None):
+        """Runs a query to retrieve at most `keys_read_max` keys and returns db result."""
+        max_keys = max_keys or self.keys_read_max
+        LOG.debug('Row limit: %d', max_keys)
+
+        selq = select(self._keys.c.publickey, self._keys.c.id, self._keys.c.email) \
+            .where(and_(self._keys.c.status == db.ST_DEFAULT, self._keys.c.confirm == db.CO_CONFIRMED)) \
+            .limit(max_keys)
+
+        LOG.debug('Retrieving keys to be processed: %s -- %s', selq, selq.compile().params)
+        with self._engine.connect() as conn:
+            return [e for e in conn.execute(selq)]
+
+    def count_keys(self):
+        selq = select(func.count(self._keys.c.id)) \
+            .where(and_(self._keys.c.status == db.ST_DEFAULT, self._keys.c.confirm == db.CO_CONFIRMED))
+
+        LOG.debug('Counting all keys: %s -- %s', selq, selq.compile().params)
+        try:
+            with self._engine.connect() as conn:
+                res = conn.execute(selq)
+                # This is a 1-element tuple.
+                return res.one_or_none()[0]
+        except OperationalError:
+            LOG.exception('Cannot count keys')
+            return None
+
+    def fetch_keys_to_delete(self):
+        seldel = select(self._keys.c.email, self._keys.c.id) \
+            .where(self._keys.c.status == db.ST_TO_BE_DELETED) \
+            .limit(self.keys_read_max)
+
+        with self._engine.connect() as conn:
+            return [e for e in conn.execute(seldel)]
+
+    def delete_keys(self, row_id, /, email=None):
+        """Remove key from the database."""
+        if email is not None:
+            LOG.debug('Deleting key: id=%s, email=%s', row_id, email)
+            delq = delete(self._keys).where(and_(self._keys.c.email == email, self._keys.c.id == row_id))
+        else:
+            LOG.debug('Deleting key: id=%s', row_id)
+            delq = delete(self._keys).where(self._keys.c.id == row_id)
+
+        with self._engine.connect() as conn:
+            LOG.debug('Deleting public keys associated with confirmed email: %s', delq)
+            conn.execute(delq)
+
+    def delete_key_by_email(self, email):
+        """Remove keys linked to the given email from the database."""
+        delq = delete(self._keys).where(self._keys.c.email == email)
+
+        LOG.debug('Deleting email for: %s', email)
+        with self._engine.connect() as conn:
+            conn.execute(delq)
+
+    def mark_accepted(self, row_id):
+        modq = self._keys.update().where(self._keys.c.id == row_id).values(status=db.ST_IMPORTED)
+        LOG.debug("Key imported, updating key: %s", modq)
+
+        with self._engine.connect() as conn:
+            conn.execute(modq)

lacre/smime.py

@@ -0,0 +1,126 @@
+#
+# lacre
+#
+# This file is part of the lacre source code.
+#
+# lacre is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# lacre source code is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""S/MIME handling module."""
+
+import os
+
+from M2Crypto import BIO, SMIME, X509
+
+import logging
+import lacre.text as text
+import lacre.config as conf
+import lacre.transport as xport
+
+LOG = logging.getLogger(__name__)
+
+
+#
+# WARNING: This file is not covered with E2E tests.
+#
+
+def encrypt(raw_message, recipients, from_addr):
+    """Encrypt with S/MIME."""
+    if not conf.config_item_set('smime', 'cert_path'):
+        LOG.info("No valid path for S/MIME certs found in config file. S/MIME encryption aborted.")
+        return recipients
+
+    cert_path = conf.get_item('smime', 'cert_path')+"/"
+    s = SMIME.SMIME()
+    sk = X509.X509_Stack()
+    smime_to = list()
+    cleartext_to = list()
+
+    for addr in recipients:
+        cert_and_email = _get_cert_for_email(addr, cert_path)
+
+        if not (cert_and_email is None):
+            (to_cert, normal_email) = cert_and_email
+            LOG.debug("Found cert " + to_cert + " for " + addr + ": " + normal_email)
+            smime_to.append(addr)
+            x509 = X509.load_cert(to_cert, format=X509.FORMAT_PEM)
+            sk.push(x509)
+        else:
+            cleartext_to.append(addr)
+
+    if smime_to:
+        s.set_x509_stack(sk)
+        s.set_cipher(SMIME.Cipher('aes_192_cbc'))
+        p7 = s.encrypt(BIO.MemoryBuffer(raw_message.as_string()))
+        # Output p7 in mail-friendly format.
+        out = BIO.MemoryBuffer()
+        out.write('From: ' + from_addr + text.EOL_S)
+        out.write('To: ' + raw_message['To'] + text.EOL_S)
+        if raw_message['Cc']:
+            out.write('Cc: ' + raw_message['Cc'] + text.EOL_S)
+        if raw_message['Bcc']:
+            out.write('Bcc: ' + raw_message['Bcc'] + text.EOL_S)
+        if raw_message['Subject']:
+            out.write('Subject: ' + raw_message['Subject'] + text.EOL_S)
+
+        if conf.config_item_equals('default', 'add_header', 'yes'):
+            out.write('X-Lacre: Encrypted by Lacre' + text.EOL_S)
+
+        s.write(out, p7)
+
+        LOG.debug(f"Sending message from {from_addr} to {smime_to}")
+
+        send_msg = xport.SendFrom(from_addr)
+        send_msg(out.read(), smime_to)
+
+    if cleartext_to:
+        LOG.debug(f"Unable to find valid S/MIME certificates for {cleartext_to}")
+
+    return cleartext_to
+
+
+def _path_comparator(insensitive: bool):
+    if insensitive:
+        return lambda filename, recipient: filename.casefold() == recipient
+    else:
+        return lambda filename, recipient: filename == recipient
+
+
+def _get_cert_for_email(to_addr, cert_path):
+    insensitive = conf.config_item_equals('default', 'mail_case_insensitive', 'yes')
+    paths_equal = _path_comparator(insensitive)
+
+    LOG.info('Retrieving certificate for %s from %s, insensitive=%s',
+             to_addr, cert_path, insensitive)
+
+    files_in_directory = os.listdir(cert_path)
+    for filename in files_in_directory:
+        file_path = os.path.join(cert_path, filename)
+        if not os.path.isfile(file_path):
+            continue
+
+        if paths_equal(file_path, to_addr):
+            return (file_path, to_addr)
+
+    # support foo+ignore@bar.com -> foo@bar.com
+    LOG.info(f"An email with topic? {to_addr}")
+    (fixed_up_email, topic) = text.parse_delimiter(to_addr)
+    LOG.info(f'Got {fixed_up_email!r} and {topic!r}')
+    if topic is None:
+        # delimiter not used
+        LOG.info('Topic not found')
+        return None
+    else:
+        LOG.info(f"Looking up certificate for {fixed_up_email} after parsing {to_addr}")
+        return _get_cert_for_email(fixed_up_email, cert_path)

lacre/stats.py

@@ -0,0 +1,29 @@
+"""Insights into Lacre's inner workings."""
+
+import time
+import logging
+
+class ExecutionTimeLogger:
+    """Context-manager that measures how much time some operation took and logs it."""
+
+    def __init__(self, message: str, logger: logging.Logger):
+        self._message = message
+        self._log = logger
+        self._start = None
+
+    def __enter__(self):
+        self._start = time.process_time()
+        self._log.info('Start: %s', self._message)
+
+    def __exit__(self, exc_type=None, exc_value=None, traceback=None):
+        end = time.process_time()
+        ellapsed = (end - self._start) * 1000
+
+        if exc_type:
+            exception = (exc_type, exc_value, traceback)
+            self._log.error('%s took %d ms, raised exception', self._message, ellapsed, exc_info=exception)
+        else:
+            self._log.info('%s took %d ms', self._message, ellapsed)
+
+def time_logger(msg: str, logger: logging.Logger):
+    return ExecutionTimeLogger(msg, logger)

lacre/text.py

@@ -1,20 +1,27 @@
+"""Basic payload-processing routines."""
+
 import sys
 import re
 import logging
+from email.message import EmailMessage
 
-import lacre.config as conf
 
 # The standard way to encode line-ending in email:
-EOL = "\r\n"
+EOL = b"\r\n"
+EOL_S = EOL.decode()
+DOUBLE_EOL_BYTES = EOL*2
 
-PGP_INLINE_BEGIN = b"-----BEGIN PGP MESSAGE-----"
-PGP_INLINE_END = b"-----END PGP MESSAGE-----"
+PGP_BEGIN = b"-----BEGIN PGP MESSAGE-----"
+PGP_END = b"-----END PGP MESSAGE-----"
+
+PGP_BEGIN_S = PGP_BEGIN.decode()
+PGP_END_S = PGP_END.decode()
 
 
 LOG = logging.getLogger(__name__)
 
 
-def parse_content_type(content_type):
+def parse_content_type(content_type: str):
     """Analyse Content-Type email header.
 
     Return a pair: type and sub-type.
@@ -49,19 +56,49 @@ def parse_delimiter(address: str):
         return (address, None)
 
 
-def sanitize_case_sense(address):
-    """Sanitize email case."""
-    # TODO: find a way to make it more unit-testable
-    if conf.flag_enabled('default', 'mail_case_insensitive'):
-        address = address.lower()
+def _lowercase_whole_address(address: str):
+    return address.lower()
+
+
+def _lowercase_domain_only(address: str):
+    parts = address.split('@', maxsplit=2)
+    if len(parts) > 1:
+        return parts[0] + '@' + parts[1].lower()
     else:
-        splitted_address = address.split('@')
-        if len(splitted_address) > 1:
-            address = splitted_address[0] + '@' + splitted_address[1].lower()
-
-    return address
+        return address
 
 
-def is_pgp_inline(payload) -> bool:
-    """Find out if the payload (bytes) contains PGP/INLINE markers."""
-    return PGP_INLINE_BEGIN in payload and PGP_INLINE_END in payload
+def choose_sanitizer(mail_case_insensitive: bool):
+    """Return a function to sanitize email case sense."""
+    if mail_case_insensitive:
+        return _lowercase_whole_address
+    else:
+        return _lowercase_domain_only
+
+
+def is_payload_pgp_inline(payload) -> bool:
+    """Find out if the payload (bytes) contains PGP/inline markers."""
+    if isinstance(payload, bytes):
+        return payload.startswith(PGP_BEGIN) and _ends_with(payload, PGP_END)
+    elif isinstance(payload, str):
+        return payload.startswith(PGP_BEGIN_S) and _ends_with(payload, PGP_END_S)
+    else:
+        raise TypeError('Expected str or bytes')
+
+
+def _ends_with(payload, marker) -> bool:
+    # Length of the span at the end of the payload we want to inspect should
+    # include CRLF, CR or LF, so make it slightly larger than the marker
+    # itself.
+    span = len(marker) + 2
+    return marker in payload[-span:]
+
+
+def is_message_pgp_inline(message: EmailMessage) -> bool:
+    """Find out if a message is already PGP-Inline encrypted."""
+    if message.is_multipart() or isinstance(message.get_payload(), list):
+        # more than one payload, check each one of them
+        return any(is_message_pgp_inline(m.payload()) for m in message.iter_parts())
+    else:
+        # one payload, check it
+        return is_payload_pgp_inline(message.get_payload(decode=True))

lacre/transport.py

@@ -0,0 +1,71 @@
+"""SMTP transport module."""
+
+import smtplib
+import logging
+from typing import AnyStr, List
+
+import lacre.config as conf
+
+# Mail status constants.
+#
+# These are the only values that our mail handler is allowed to return.
+RESULT_OK = '250 OK'
+RESULT_ERROR = '500 Could not process your message'
+
+LOG = logging.getLogger(__name__)
+
+# This is a left-over from old architecture.
+from_addr = None
+
+
+def register_sender(fromaddr):
+    """Set module state: message sender address."""
+    global from_addr
+    LOG.warning('Setting global recipient: %s', fromaddr)
+    from_addr = fromaddr
+
+
+def send_msg(message: AnyStr, recipients: List[str]):
+    """Send MESSAGE to RECIPIENTS to the mail relay."""
+    global from_addr
+    LOG.debug('Delivery from %s to %s', from_addr, recipients)
+
+    recipients = [_f for _f in recipients if _f]
+    if recipients:
+        LOG.info(f"Sending email to: {recipients!r}")
+        relay = conf.relay_params()
+        smtp = smtplib.SMTP(relay[0], relay[1])
+        if conf.flag_enabled('relay', 'starttls'):
+            smtp.starttls()
+        smtp.sendmail(from_addr, recipients, message)
+    else:
+        LOG.info("No recipient found")
+
+
+class SendFrom:
+    """A class wrapping the transport process."""
+
+    def __init__(self, from_addr):
+        """Initialise the transport."""
+        self._from_addr = from_addr
+
+    def __call__(self, message: AnyStr, recipients: List[str]):
+        """Send the given message to all recipients from the list.
+
+        - Message is the email object serialised to str or bytes.
+        - Empty recipients are filtered out before communication.
+        """
+        recipients = [_f for _f in recipients if _f]
+
+        if not recipients:
+            LOG.warning("No recipient found")
+            return
+
+        LOG.info("Sending email to: %s", recipients)
+        relay = conf.relay_params()
+        smtp = smtplib.SMTP(relay[0], relay[1])
+
+        if conf.flag_enabled('relay', 'starttls'):
+            smtp.starttls()
+
+        smtp.sendmail(self._from_addr, recipients, message)

requirements.txt

@@ -1,5 +1,5 @@
+aiosmtpd==1.4.2
 SQLAlchemy==1.4.32
 Markdown==3.4.1
 M2Crypto==0.38.0
 requests==2.27.1
-watchdog==2.1.9

test/daemon_test.py

@@ -1,20 +1,20 @@
 #
-# gpg-mailgate
+# lacre
 #
-# This file is part of the gpg-mailgate source code.
+# This file is part of the lacre source code.
 #
-# gpg-mailgate is free software: you can redistribute it and/or modify
+# lacre is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# gpg-mailgate source code is distributed in the hope that it will be useful,
+# lacre source code is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with gpg-mailgate source code. If not, see <http://www.gnu.org/licenses/>.
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
 #
 
 import configparser
@@ -22,13 +22,15 @@ import logging
 import subprocess
 import os
 import time
+import unittest
 
 
 def _spawn(cmd):
     env_dict = {
         "PATH": os.getenv("PATH"),
         "PYTHONPATH": os.getcwd(),
-        "GPG_MAILGATE_CONFIG": "test/gpg-mailgate-daemon-test.conf"
+        "LANG": 'en_US.UTF-8',
+        "LACRE_CONFIG": "test/lacre-daemon.conf"
         }
     logging.debug(f"Spawning command: {cmd} with environment: {env_dict!r}")
     return subprocess.Popen(cmd,
@@ -38,25 +40,19 @@ def _spawn(cmd):
 
 
 def _interrupt(proc):
-    # proc.send_signal(signal.SIGINT)
     proc.terminate()
 
 
-def _load(name):
-    logging.debug(f"Loading file {name}")
-    f = open(name, "r")
-    contents = f.read()
-    f.close()
-    return contents
-
-
 def _send(host, port, mail_from, mail_to, message):
     logging.debug(f"Sending message to {host}:{port}")
-    _spawn([os.getenv("PYTHON") or "python",
-            "test/utils/sendmail.py",
-            "-f", mail_from,
-            "-t", mail_to,
-            "-m", message])
+    p = _spawn([os.getenv("PYTHON") or "python",
+        "test/utils/sendmail.py",
+        "-f", mail_from,
+        "-t", mail_to,
+        "-m", message])
+
+    # Perform subprocess's internal resource management:
+    p.communicate()
 
 
 def _load_test_config():
@@ -65,53 +61,76 @@ def _load_test_config():
     return cp
 
 
-def _report_result(message_file, expected, test_output):
-    status = None
-    if expected in test_output:
-        status = "Success"
-    else:
-        status = "Failure"
+class AdvancedMailFilterE2ETest(unittest.TestCase):
+    """End-to-end tests for Advanced Mail Filter.
 
-    print(message_file.ljust(30), status)
+    These tests are described by e2e.ini file, each case being a
+    separate section.  All cases are executed following the same
+    procedure:
+    1. start up a mail relay mock;
+    2. load test message;
+    3. send the message to the daemon;
+    4. check if message received by relay mock meets criteria.
+
+    Before any case is executed, the daemon is started and finally it's
+    terminated by sending it a SIGINT signal."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Start up the daemon."""
+        cls.config = _load_test_config()
+
+        python = os.getenv("PYTHON", "python")
+
+        logging.info('Starting the server...')
+        cls.server = _spawn([python, '-m', 'lacre.daemon'])
+
+    @classmethod
+    def tearDownClass(cls):
+        """Terminate the daemon."""
+        logging.info('Closing the server (SIGINT): %s', (cls.server))
+        _interrupt(cls.server)
+
+    def case_names(self):
+        """A generator yielding a sequence of test case names."""
+        def is_test_case(case_name: str) -> bool:
+            return case_name.startswith('case-')
+
+        for tc in filter(is_test_case, self.config.sections()):
+            yield tc
+
+    def test_all_cases(self):
+        for case_name in self.case_names():
+            with self.subTest(case=case_name):
+                self._execute_case(self.config, case_name=case_name)
+
+    def _execute_case(self, config, case_name):
+        logging.info(f"Executing case {case_name}")
+        python = os.getenv("PYTHON", "python")
+
+        relay_mock = _spawn([python, "test/utils/relay.py", "2500"])
+        time.sleep(1)                            # Wait for the relay to start up.
+
+        _send("localhost", 10025, "dave@disposlab",
+              config.get(case_name, 'to'), config.get(case_name, 'in'))
+
+        (test_out, _) = relay_mock.communicate()
+
+        test_out = test_out.decode('utf-8')
+        logging.debug(f"Read {len(test_out)} characters of output: '{test_out}'")
+
+        if 'out' in config[case_name]:
+            expected = '\r\n' + self.config.get(case_name, 'out')
+            self.assertIn(expected, test_out, self.config.get(case_name, 'in'))
+        else:
+            unexpected = '\r\n' + self.config.get(case_name, 'out-not')
+            self.assertNotIn(unexpected, test_out, self.config.get(case_name, 'in'))
 
 
-def _execute_case(config, case_name):
-    logging.info(f"Executing case {case_name}")
-    python = os.getenv("PYTHON", "python")
-
-    relay_mock = _spawn([python, "test/utils/relay.py", "2500"])
-    time.sleep(1)                            # Wait for the relay to start up.
-
-    _send("localhost", 10025, "dave@disposlab",
-          config.get(case_name, 'to'), config.get(case_name, 'in'))
-
-    relay_mock.wait()
-    (test_out, _) = relay_mock.communicate()
-
-    test_out = test_out.decode('utf-8')
-    logging.debug(f"Read {len(test_out)} characters of output: '{test_out}'")
-
-    _report_result(config.get(case_name, "in"), config.get(case_name, "out"), test_out)
-
-
-def _main():
-    conf = _load_test_config()
-
+if __name__ == '__main__':
     logging.basicConfig(filename="test/logs/daemon-test.log",
                         format="%(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s",
                         datefmt="%Y-%m-%d %H:%M:%S",
                         level=logging.DEBUG)
 
-    logging.info("Starting Lacre Daemon tests...")
-    python = os.getenv("PYTHON", "python")
-
-    server = _spawn([python, "-m", "lacre.daemon"])
-
-    for case_no in range(1, conf.getint("tests", "cases")):
-        _execute_case(conf, case_name=f"case-{case_no}")
-
-    _interrupt(server)
-
-
-if __name__ == '__main__':
-    _main()
+    unittest.main()

test/e2e.ini

@@ -29,13 +29,19 @@ keys: test/keyhome
 certs: test/certs
 
 [tests]
-# Number of "test-*" sections in this file, describing test cases.
-cases: 8
 e2e_log: test/logs/e2e.log
 e2e_log_format: %(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s
 e2e_log_datefmt: %Y-%m-%d %H:%M:%S
-lacre_log: test/logs/gpg-mailgate.log
-log_config: test/gpg-lacre-log.ini
+lacre_log: test/logs/lacre-simple.log
+log_config: test/lacre-logging.conf
+
+# TEST IDENTITIES AND SETTINGS:
+#
+# Email             Key       Style
+# alice@disposlab   RSA 3072  PGP/Inline
+# bob@disposlab     ED25519   PGP/Inline
+# carlos@disposlab  none      PGP/Inline
+# evan@disposlab    ED25519   PGP/MIME
 
 [case-1]
 descr: Clear text message to a user without a key
@@ -80,7 +86,91 @@ in: test/msgin/clear2rsa2.msg
 out: -----BEGIN PGP MESSAGE-----
 
 [case-8]
-descr: Clear text message to address with delimiter and a user with an Ed25519 key.
-to: bob@disposlab
+descr: Clear text message to address with delimiter and a user with an Ed25519 key
+to: bob+foo@disposlab
 in: test/msgin/clear2ed-delim.msg
 out: -----BEGIN PGP MESSAGE-----
+
+[case-9]
+descr: Clear text message with inline PGP markers to recipient with a key
+to: bob@disposlab
+in: test/msgin/with-markers2clear.msg
+out-not: This message includes inline PGP markers.
+
+[case-10]
+descr: UTF-8 message (yields Base64)
+to: carlos@disposlab
+in: test/msgin/utf8.msg
+out: xYHEhENaTk/FmsSGLiBaYcW6w7PFgsSHIGfEmcWbbMSFIGphxbrFhC4=
+
+[case-11]
+descr: Non-ASCII message (ISO-8859-2; yields quoted-printable)
+to: carlos@disposlab
+in: test/msgin/nonascii.msg
+out: =A3=A1CZNO=A6=C6.
+
+[case-12]
+descr: multipart/alternative with UTF-8, not encrypted
+to: carlos@disposlab
+in: test/msgin/utf8-alternative.msg
+out-not: -----BEGIN PGP MESSAGE-----
+
+[case-13]
+descr: multipart/alternative with UTF-8, encrypted
+to: evan@disposlab
+in: test/msgin/utf8-alternative.msg
+out: -----BEGIN PGP MESSAGE-----
+
+[case-14]
+descr: Clear text with UTF-8, PGP/MIME
+to: evan@disposlab
+in: test/msgin/utf8-plain.msg
+out: Content-Type: application/pgp-encrypted
+
+[case-15]
+descr: Clear text with UTF-8, PGP/Inline
+to: bob@disposlab
+in: test/msgin/utf8-plain.msg
+out: -----BEGIN PGP MESSAGE-----
+
+[case-16]
+descr: HTML, cleartext
+to: carlos@disposlab
+in: test/msgin/html-utf8.msg
+out: PGh0bWw+DQo8aGVhZD4NCjwvaGVhZD4NCjxib2R5Pg0KWkHFu8OTxYHEhiBHxJjFmkzEhCBKQcW5
+
+[case-17]
+descr: HTML, PGP/MIME
+to: evan@disposlab
+in: test/msgin/html-utf8.msg
+out: -----BEGIN PGP MESSAGE-----
+
+[case-18]
+descr: HTML, PGP/Inline
+to: bob@disposlab
+in: test/msgin/html-utf8.msg
+out: -----BEGIN PGP MESSAGE-----
+
+[case-19]
+descr: US-ASCII HTML, cleartext
+to: carlos@disposlab
+in: test/msgin/html-ascii.msg
+out: <html>
+
+[case-20]
+descr: US-ASCII HTML, PGP/Inline
+to: bob@disposlab
+in: test/msgin/html-ascii.msg
+out: -----BEGIN PGP MESSAGE-----
+
+[case-21]
+descr: US-ASCII HTML, PGP/MIME
+to: evan@disposlab
+in: test/msgin/html-ascii.msg
+out: -----BEGIN PGP MESSAGE-----
+
+[case-22]
+descr: HTML with emoji, PGP/MIME
+to: evan@disposlab
+in: test/msgin/emoji.msg
+out: -----BEGIN PGP MESSAGE-----

test/e2e_test.py

@@ -1,20 +1,20 @@
 #
-# gpg-mailgate
+# lacre
 #
-# This file is part of the gpg-mailgate source code.
+# This file is part of the lacre source code.
 #
-# gpg-mailgate is free software: you can redistribute it and/or modify
+# lacre is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# gpg-mailgate source code is distributed in the hope that it will be useful,
+# lacre source code is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with gpg-mailgate source code. If not, see <http://www.gnu.org/licenses/>.
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
 #
 
 import os
@@ -23,9 +23,11 @@ import subprocess
 import configparser
 import logging
 
+import unittest
+
 
 RELAY_SCRIPT = "test/utils/relay.py"
-CONFIG_FILE = "test/gpg-mailgate.conf"
+CONFIG_FILE = "test/lacre.conf"
 
 
 def _build_config(config):
@@ -37,6 +39,11 @@ def _build_config(config):
     cp.add_section("gpg")
     cp.set("gpg", "keyhome", config["gpg_keyhome"])
 
+    cp.add_section('database')
+    cp.set('database', 'enabled', 'yes')
+    cp.set('database', 'url', 'sqlite:///test/lacre.db')
+    cp.set('database', 'pooling_mode', 'optimistic')
+
     cp.add_section("smime")
     cp.set("smime", "cert_path", config["smime_certpath"])
 
@@ -58,7 +65,6 @@ def _build_config(config):
     # uses PGP/MIME.
     cp.set("pgp_style", "evan@disposlab", "mime")
 
-    logging.debug(f"Created config with keyhome={config['gpg_keyhome']}, cert_path={config['smime_certpath']} and relay at port {config['port']}")
     return cp
 
 
@@ -74,64 +80,11 @@ def _write_test_config(outfile, **config):
 
 
 def _load_file(name):
-    f = open(name, 'r')
+    f = open(name, 'rb')
     contents = f.read()
     f.close()
 
-    return bytes(contents, 'utf-8')
-
-
-def _report_result(message_file, expected, test_output):
-    status = None
-    if expected in test_output:
-        status = "Success"
-    else:
-        status = "Failure"
-
-    print(message_file.ljust(30), status)
-
-
-def _execute_e2e_test(case_name, config, config_path):
-    """Read test case configuration from config and run that test case.
-
-    Parameter case_name should refer to a section in test
-    config file.  Each of these sections should contain
-    following properties: 'descr', 'to', 'in' and 'out'.
-    """
-    # This environment variable is set in Makefile.
-    python_path = os.getenv('PYTHON', 'python3')
-
-    gpglacre_cmd = [python_path,
-                    "gpg-mailgate.py",
-                    config.get(case_name, "to")]
-
-    relay_cmd = [python_path,
-                 config.get("relay", "script"),
-                 config.get("relay", "port")]
-
-    logging.debug(f"Spawning relay: {relay_cmd}")
-    relay_proc = subprocess.Popen(relay_cmd,
-                                  stdin=None,
-                                  stdout=subprocess.PIPE)
-
-    logging.debug(f"Spawning GPG-Lacre: {gpglacre_cmd}, stdin = {config.get(case_name, 'in')}")
-
-    # pass PATH because otherwise it would be dropped
-    gpglacre_proc = subprocess.run(gpglacre_cmd,
-                                   input=_load_file(config.get(case_name, "in")),
-                                   capture_output=True,
-                                   env={"GPG_MAILGATE_CONFIG": config_path,
-                                        "PATH": os.getenv("PATH")})
-
-    # Let the relay process the data.
-    relay_proc.wait()
-
-    (testout, _) = relay_proc.communicate()
-    testout = testout.decode('utf-8')
-
-    logging.debug(f"Read {len(testout)} characters of test output: '{testout}'")
-
-    _report_result(config.get(case_name, "in"), config.get(case_name, "out"), testout)
+    return contents
 
 
 def _load_test_config():
@@ -141,28 +94,92 @@ def _load_test_config():
     return cp
 
 
-config = _load_test_config()
+class SimpleMailFilterE2ETest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls._e2e_config = _load_test_config()
+        cls._e2e_config_path = os.path.join(os.getcwd(), CONFIG_FILE)
 
-logging.basicConfig(filename	= config.get("tests", "e2e_log"),
-                    # Get raw values of log and date formats because they
-                    # contain %-sequences and we don't want them to be expanded
-                    # by the ConfigParser.
-                    format		= config.get("tests", "e2e_log_format", raw=True),
-                    datefmt		= config.get("tests", "e2e_log_datefmt", raw=True),
-                    level		= logging.DEBUG)
+        # This environment variable is set in Makefile.
+        cls._python_path = os.getenv('PYTHON', 'python3')
 
-config_path = os.getcwd() + "/" + CONFIG_FILE
+        _write_test_config(cls._e2e_config_path,
+                           port				= cls._e2e_config.get("relay", "port"),
+                           gpg_keyhome		= cls._e2e_config.get("dirs", "keys"),
+                           smime_certpath	= cls._e2e_config.get("dirs", "certs"),
+                           log_config		= cls._e2e_config.get("tests", "log_config"))
 
-_write_test_config(config_path,
-                  port				= config.get("relay", "port"),
-                  gpg_keyhome		= config.get("dirs", "keys"),
-                  smime_certpath	= config.get("dirs", "certs"),
-                  log_config		= config.get("tests", "log_config"))
+    def case_names(self):
+        def is_test_case(case_name: str) -> bool:
+            return case_name.startswith('case-')
 
-for case_no in range(1, config.getint("tests", "cases")+1):
-    case_name = f"case-{case_no}"
-    logging.info(f"Executing {case_name}: {config.get(case_name, 'descr')}")
+        for tc in filter(is_test_case, self._e2e_config.sections()):
+            yield tc
 
-    _execute_e2e_test(case_name, config, config_path)
+    def test_all_cases(self):
+        for case_name in self.case_names():
+            with self.subTest(case=case_name):
+                self._execute_e2e_test(case_name)
 
-print("See diagnostic output for details. Tests: '%s', Lacre: '%s'" % (config.get("tests", "e2e_log"), config.get("tests", "lacre_log")))
+    def _execute_e2e_test(self, case_name):
+        """Read test case configuration from config and run that test case.
+
+        Parameter case_name should refer to a section in test
+        config file.  Each of these sections should contain
+        following properties: 'descr', 'to', 'in' and 'out'.
+        """
+        gpglacre_cmd = self._python_command(
+            'lacre.py',
+            self._e2e_config.get(case_name, 'to'))
+
+        relay_cmd = self._python_command(
+            self._e2e_config.get("relay", "script"),
+            self._e2e_config.get("relay", "port"))
+
+        logging.debug(f"Spawning relay: {relay_cmd}")
+        relay_proc = subprocess.Popen(relay_cmd,
+            stdin=None,
+            stdout=subprocess.PIPE)
+
+        logging.debug(f"Spawning GPG-Lacre: {gpglacre_cmd}, stdin = {self._e2e_config.get(case_name, 'in')}")
+
+        # pass PATH because otherwise it would be dropped
+        gpglacre_proc = subprocess.run(gpglacre_cmd,
+            input=_load_file(self._e2e_config.get(case_name, "in")),
+            capture_output=True,
+            env={"LACRE_CONFIG": self._e2e_config_path,
+                "PATH": os.getenv("PATH")})
+
+        # Let the relay process the data.
+        relay_proc.wait()
+
+        (testout, _) = relay_proc.communicate()
+        testout = testout.decode('utf-8')
+
+        logging.debug(f"Read {len(testout)} characters of test output: '{testout}'")
+
+        if 'out' in self._e2e_config[case_name]:
+            expected = "\r\n" + self._e2e_config.get(case_name, "out")
+            self.assertIn(expected, testout, self._e2e_config.get(case_name, "in"))
+        else:
+            unexpected = "\r\n" + self._e2e_config.get(case_name, "out-not")
+            self.assertNotIn(unexpected, testout, self._e2e_config.get(case_name, "in"))
+
+    def _python_command(self, script, *args):
+        command = [self._python_path, script]
+        command.extend(args)
+        return command
+
+
+if __name__ == '__main__':
+    config = _load_test_config()
+
+    logging.basicConfig(filename	= config.get("tests", "e2e_log"),
+                        # Get raw values of log and date formats because they
+                        # contain %-sequences and we don't want them to be
+                        # expanded by the ConfigParser.
+                        format		= config.get("tests", "e2e_log_format", raw=True),
+                        datefmt		= config.get("tests", "e2e_log_datefmt", raw=True),
+                        level		= logging.DEBUG)
+
+    unittest.main()

test/keys/bob@disposlab.pub

@@ -0,0 +1,13 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mDMEYdTFkRYJKwYBBAHaRw8BAQdA2tgdP1pMt3cv3XAW7ov5AFn74mMZvyTksp9Q
+eO1PkpK0GkJvYiBGb29iYXIgPGJvYkBkaXNwb3NsYWI+iJYEExYIAD4WIQQZz0tH
+7MnEevqE1L2W85/aDjG7ZwUCYdTFkQIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgID
+AQIeAQIXgAAKCRCW85/aDjG7ZxVnAP49t7BU2H+/WCpa3fCAlMEcik82sU4p+U9D
+pMsbjawwYgEA1SbA5CF835cMjoEufy1h+2M4T9gI/0X2lk8OAtwwggm4OARh1MXg
+EgorBgEEAZdVAQUBAQdAUVNKx2OsGtNdRsnl3J/uv6obkUC0KcO4ikdRs+iejlMD
+AQgHiHgEGBYIACAWIQQZz0tH7MnEevqE1L2W85/aDjG7ZwUCYdTF4AIbDAAKCRCW
+85/aDjG7Z039APwLGP5ibqCC9yIr4YVbdWff1Ch+2C91MR2ObF93Up9+ogD8D2zd
+OjjB6xRD0Q2FN+alsNGCtdutAs18AZ5l33RMzws=
+=wWoq
+-----END PGP PUBLIC KEY BLOCK-----

test/lacre-cron.conf

@@ -1,30 +1,32 @@
 [logging]
-config = test/gpg-lacre-log.ini
+config = test/lacre-logging.conf
 file = test/logs/gpg-mailgate.log
 format = %(asctime)s %(module)s[%(process)d]: %(message)s
 date_format = ISO
 
 [gpg]
 keyhome = test/keyhome
-cache_refresh_minutes = 1
 
 [smime]
 cert_path = test/certs
 
+[daemon]
+host = not_used
+port = not_used
+
 [database]
 enabled = yes
 url = sqlite:///test/lacre.db
+pooling_mode = optimistic
+max_connection_age = 3600
 
 [relay]
 host = localhost
 port = 2500
 
-[daemon]
-host = localhost
-port = 10025
-
 [cron]
 send_email = no
+mail_templates = not_used
 
 [enc_keymap]
 alice@disposlab = 1CD245308F0963D038E88357973CF4D9387C44D7

test/lacre-daemon.conf

@@ -1,5 +1,5 @@
 [logging]
-config = test/gpg-lacre-log.ini
+config = test/lacre-logging.conf
 file = test/logs/gpg-mailgate.log
 format = %(asctime)s %(module)s[%(process)d]: %(message)s
 date_format = ISO
@@ -13,15 +13,26 @@ cert_path = test/certs
 [database]
 enabled = yes
 url = sqlite:///test/lacre.db
+pooling_mode = optimistic
 
 [relay]
 host = localhost
 port = 2500
 
+[daemon]
+host = localhost
+port = 10025
+log_headers = yes
+
 [cron]
 send_email = no
+mail_templates = not_used
+
+[pgp_style]
+# this recipient has PGP/MIME enabled, because the default approach is to use
+# PGP/Inline
+evan@disposlab = mime
 
 [enc_keymap]
 alice@disposlab = 1CD245308F0963D038E88357973CF4D9387C44D7
 bob@disposlab = 19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67
-

test/lacre-logging.conf

@@ -18,7 +18,7 @@ formatter=postfixfmt
 args=('test/logs/lacre.log', 'a+')
 
 [formatter_postfixfmt]
-format=%(asctime)s %(module)s[%(process)d]: %(message)s
+format=%(asctime)s %(name)s[%(process)d]: %(message)s
 datefmt=%b %e %H:%M:%S
 style=%
 validate=True

test/modules/test_contracts.py

@@ -1,20 +1,20 @@
 #
-#	gpg-mailgate
+# lacre
 #
-#	This file is part of the gpg-mailgate source code.
+# This file is part of the lacre source code.
 #
-#	gpg-mailgate is free software: you can redistribute it and/or modify
-#	it under the terms of the GNU General Public License as published by
-#	the Free Software Foundation, either version 3 of the License, or
-#	(at your option) any later version.
+# lacre is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
 #
-#	gpg-mailgate source code is distributed in the hope that it will be useful,
-#	but WITHOUT ANY WARRANTY; without even the implied warranty of
-#	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-#	GNU General Public License for more details.
+# lacre source code is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
 #
-#	You should have received a copy of the GNU General Public License
-#	along with gpg-mailgate source code. If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU General Public License
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
 #
 
 """Unit-tests as contracts for external dependencies.
@@ -25,68 +25,191 @@ documentation.
 """
 
 import email
+import email.mime.multipart
+from email.message import EmailMessage
+from email.policy import SMTP, SMTPUTF8
+from email.errors import HeaderParseError
+
 import unittest
 from configparser import RawConfigParser
 
+
 class EmailParsingTest(unittest.TestCase):
-	"""This test serves as a package contract and documentation of its behaviour."""
+    """This test serves as a package contract and documentation of its behaviour."""
 
-	def test_message_from_bytes_produces_message_with_str_headers(self):
-		rawmsg = b"From: alice@lacre.io\r\n" \
-		  + b"To: bob@lacre.io\r\n" \
-		  + b"Subject: Test message\r\n" \
-		  + b"\r\n" \
-		  + b"Test message from Alice to Bob.\r\n"
+    def test_message_from_bytes_produces_message_with_str_headers(self):
+        rawmsg = b"From: alice@lacre.io\r\n" \
+          + b"To: bob@lacre.io\r\n" \
+          + b"Subject: Test message\r\n" \
+          + b"\r\n" \
+          + b"Test message from Alice to Bob.\r\n"
 
-		parsed = email.message_from_bytes(rawmsg)
+        parsed = email.message_from_bytes(rawmsg)
 
-		self.assertEqual(parsed["From"], "alice@lacre.io")
-		self.assertEqual(parsed["To"], "bob@lacre.io")
-		self.assertEqual(parsed["Subject"], "Test message")
+        self.assertEqual(parsed["From"], "alice@lacre.io")
+        self.assertEqual(parsed["To"], "bob@lacre.io")
+        self.assertEqual(parsed["Subject"], "Test message")
 
-	def test_bytes_message_payload_decoded_produces_bytes(self):
-		rawmsg = b"From: alice@lacre.io\r\n" \
-		  + b"To: bob@lacre.io\r\n" \
-		  + b"Subject: Test message\r\n" \
-		  + b"\r\n" \
-		  + b"Test message from Alice to Bob.\r\n"
+    def test_bytes_message_payload_decoded_produces_bytes(self):
+        rawmsg = b"From: alice@lacre.io\r\n" \
+          + b"To: bob@lacre.io\r\n" \
+          + b"Subject: Test message\r\n" \
+          + b"\r\n" \
+          + b"Test message from Alice to Bob.\r\n"
 
-		parsed = email.message_from_bytes(rawmsg)
+        parsed = email.message_from_bytes(rawmsg)
 
-		self.assertEqual(parsed.get_payload(), "Test message from Alice to Bob.\r\n")
-		self.assertEqual(parsed.get_payload(decode=True), b"Test message from Alice to Bob.\r\n")
+        self.assertEqual(parsed["From"], "alice@lacre.io")
+        self.assertEqual(parsed.get_payload(), "Test message from Alice to Bob.\r\n")
+        self.assertEqual(parsed.get_payload(decode=True), b"Test message from Alice to Bob.\r\n")
 
-	def test_message_from_string_produces_message_with_str_headers(self):
-		rawmsg = "From: alice@lacre.io\r\n" \
-		  + "To: bob@lacre.io\r\n" \
-		  + "Subject: Test message\r\n" \
-		  + "\r\n" \
-		  + "Test message from Alice to Bob.\r\n"
+    def test_message_from_string_produces_message_with_str_headers(self):
+        rawmsg = "From: alice@lacre.io\r\n" \
+          + "To: bob@lacre.io\r\n" \
+          + "Subject: Test message\r\n" \
+          + "\r\n" \
+          + "Test message from Alice to Bob.\r\n"
 
-		parsed = email.message_from_string(rawmsg)
+        parsed = email.message_from_string(rawmsg)
 
-		self.assertEqual(parsed["From"], "alice@lacre.io")
-		self.assertEqual(parsed["To"], "bob@lacre.io")
-		self.assertEqual(parsed["Subject"], "Test message")
+        self.assertEqual(parsed["From"], "alice@lacre.io")
+        self.assertEqual(parsed["To"], "bob@lacre.io")
+        self.assertEqual(parsed["Subject"], "Test message")
 
-	def test_str_message_payload_decoded_produces_bytes(self):
-		rawmsg = "From: alice@lacre.io\r\n" \
-		  + "To: bob@lacre.io\r\n" \
-		  + "Subject: Test message\r\n" \
-		  + "\r\n" \
-		  + "Test message from Alice to Bob.\r\n"
+    def test_str_base64_payload(self):
+        rawmsg = "From: alice@lacre.io\r\n" \
+          + "To: bob@lacre.io\r\n" \
+          + "Subject: Test message\r\n" \
+          + "Content-Type: text/plain\r\n" \
+          + "Content-Transfer-Encoding: base64\r\n" \
+          + "\r\n" \
+          + "VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n"
 
-		parsed = email.message_from_string(rawmsg)
+        parsed: EmailMessage = email.message_from_string(rawmsg, policy=SMTP)
+
+        self.assertEqual(parsed.get_payload(decode=False),
+                         "VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n")
+        self.assertEqual(parsed.get_payload(decode=True),
+                         b"Test message from Alice to Bob.\n")
+        self.assertEqual(parsed.get_content(),
+                         "Test message from Alice to Bob.\n")
+
+    def test_bytes_base64_payload(self):
+        rawmsg = b"From: alice@lacre.io\r\n" \
+          + b"To: bob@lacre.io\r\n" \
+          + b"Subject: Test message\r\n" \
+          + b"Content-Type: application/octet-stream\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n"
+
+        parsed: EmailMessage = email.message_from_bytes(rawmsg, policy=SMTP)
+
+        self.assertEqual(parsed.get_payload(decode=False),
+                         "VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n")
+        self.assertEqual(parsed.get_payload(decode=True),
+                         b"Test message from Alice to Bob.\n")
+        self.assertEqual(parsed.get_content(),
+                         b"Test message from Alice to Bob.\n")
+
+    def test_multipart_parser(self):
+        rawmsg = b"Content-Type: multipart/mixed; boundary=XXXXXXXX\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX\r\n" \
+          + b"Content-Type: application/octet-stream\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"VGVzdCBtZXNzYWdlIGZyb20gQWxpY2UgdG8gQm9iLgo=\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX\r\n" \
+          + b"Content-Type: application/octet-stream\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"SGVsbG8sIFdvcmxkIQo=\r\n" \
+          + b"\r\n" \
+          + b"--XXXXXXXX--\r\n"
+
+        parsed: EmailMessage = email.message_from_bytes(rawmsg, policy=SMTP)
+
+        self.assertRaises(KeyError, parsed.get_content)
+        self.assertEqual(parsed.get_payload(0).get_content(),
+                         b'Test message from Alice to Bob.\n')
+        self.assertEqual(parsed.get_payload(1).get_content(),
+                         b'Hello, World!\n')
+
+    def test_headers_only_returns_bytes_payload(self):
+        rawmsg = b"From: alice@lacre.io\r\n" \
+          + b"To: bob@lacre.io\r\n" \
+          + b"Subject: Test message\r\n" \
+          + b"Content-Type: text/plain\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"\r\n" \
+          + b"SGVsbG8sIFdvcmxkIQo=\r\n"
+
+        parser = email.parser.BytesHeaderParser()
+        parsed = parser.parsebytes(rawmsg)
+
+        self.assertEqual(parsed.get_payload(decode=False), "SGVsbG8sIFdvcmxkIQo=\r\n")
+        self.assertEqual(parsed.get_payload(decode=True), b"Hello, World!\n")
+
+    def test_headers_only_produces_single_payload_for_multipart(self):
+        msg = None
+        with open('test/msgin/utf8-alternative.msg', 'rb') as f:
+            p = email.parser.BytesHeaderParser()
+            msg = p.parse(f)
+
+        payload = msg.get_payload()
+
+        # Taken from test/msgin/utf8-alternative.msg:
+        message_boundary = '6s7R3c0y2W8qiD7cU3iWyXcw'
+
+        self.assertIsInstance(payload, str)
+        self.assertTrue(message_boundary in payload)
+
+    def test_fail_if_message_id_parsing_is_fixed(self):
+        # Unfortunately, Microsoft sends messages with Message-Id header values
+        # that email parser can't process.
+        #
+        # Bug: https://github.com/python/cpython/issues/105802
+        # Fix: https://github.com/python/cpython/pull/108133
+
+        rawmsg = b"From: alice@lacre.io\r\n" \
+          + b"To: bob@lacre.io\r\n" \
+          + b"Subject: Test message\r\n" \
+          + b"Content-Type: text/plain\r\n" \
+          + b"Content-Transfer-Encoding: base64\r\n" \
+          + b"Message-Id: <[yada-yada-yada@microsoft.com]>\r\n" \
+          + b"\r\n" \
+          + b"SGVsbG8sIFdvcmxkIQo=\r\n"
+
+        msg = email.message_from_bytes(rawmsg, policy=SMTPUTF8)
+        self.assertEqual(len(msg.defects), 0)
+        self.assertRaises(IndexError, lambda: msg['Message-Id'])
+
+
+class EmailTest(unittest.TestCase):
+    def test_boundary_generated_after_as_string_call(self):
+        mp = email.mime.multipart.MIMEMultipart()
+        self.assertTrue(mp.get_boundary() is None)
+        _ = mp.as_string()
+        self.assertFalse(mp.get_boundary() is None)
+
+    def test_content_type_params_include_mime_type(self):
+        p = email.message.MIMEPart()
+        p.set_type('text/plain')
+        p.set_param('charset', 'UTF-8')
+        p.set_param('format', 'flowed')
+
+        self.assertIn(('text/plain', ''), p.get_params())
 
-		self.assertEqual(parsed.get_payload(), "Test message from Alice to Bob.\r\n")
-		self.assertEqual(parsed.get_payload(decode=True), b"Test message from Alice to Bob.\r\n")
 
 class RawConfigParserTest(unittest.TestCase):
-	def test_config_parser_returns_str(self):
-		cp = RawConfigParser()
-		cp.read("test/sample.ini")
-		self.assertEqual(cp.get("foo", "bar"), "quux")
-		self.assertEqual(cp.get("foo", "baz"), "14")
+    def test_config_parser_returns_str(self):
+        cp = RawConfigParser()
+        cp.read("test/sample.ini")
+        self.assertEqual(cp.get("foo", "bar"), "quux")
+        self.assertEqual(cp.get("foo", "baz"), "14")
+
 
 if __name__ == '__main__':
-	unittest.main()
+    unittest.main()

test/modules/test_gnupg.py

@@ -1,16 +1,96 @@
 import GnuPG
+import logging
 
 import unittest
 
 class GnuPGUtilitiesTest(unittest.TestCase):
-	def test_build_default_command(self):
-		cmd = GnuPG._build_command("test/keyhome")
-		self.assertEqual(cmd, ["gpg", "--homedir", "test/keyhome"])
+    def setUp(self):
+        # Record GnuPG logs:
+        logging.basicConfig(filename='test/logs/unittest.log', level=logging.DEBUG,
+                            format='%(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s')
 
-	def test_build_command_extended_with_args(self):
-		cmd = GnuPG._build_command("test/keyhome", "--foo", "--bar")
-		self.assertEqual(cmd, ["gpg", "--homedir", "test/keyhome", "--foo", "--bar"])
+    def test_build_default_command(self):
+        cmd = GnuPG._build_command("test/keyhome")
+        self.assertEqual(cmd, ["gpg", "--homedir", "test/keyhome"])
+
+    def test_build_command_extended_with_args(self):
+        cmd = GnuPG._build_command("test/keyhome", "--foo", "--bar")
+        self.assertEqual(cmd, ["gpg", "--homedir", "test/keyhome", "--foo", "--bar"])
+
+    def test_key_confirmation_with_matching_email(self):
+        armored_key = self._load('test/keys/bob@disposlab.pub')
+        matching_email = 'bob@disposlab'
+
+        is_confirmed = GnuPG.confirm_key(armored_key, matching_email)
+        self.assertTrue(is_confirmed)
+
+    def test_key_confirmation_email_mismatch(self):
+        armored_key = self._load('test/keys/bob@disposlab.pub')
+        not_matching_email = 'lucy@disposlab'
+
+        is_confirmed = GnuPG.confirm_key(armored_key, not_matching_email)
+        self.assertFalse(is_confirmed)
+
+    def test_key_listing(self):
+        keys = GnuPG.public_keys('test/keyhome')
+
+        known_identities = {
+            '1CD245308F0963D038E88357973CF4D9387C44D7': 'alice@disposlab',
+            '19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67': 'bob@disposlab',
+            '530B1BB2D0CC7971648198BBA4774E507D3AF5BC': 'evan@disposlab'
+            }
+
+        self.assertDictEqual(keys, known_identities)
+
+    def test_add_delete_key(self):
+        self.assertDictEqual(GnuPG.public_keys('test/keyhome.other'), { })
+        GnuPG.add_key('test/keyhome.other', self._load('test/keys/bob@disposlab.pub'))
+        self.assertDictEqual(GnuPG.public_keys('test/keyhome.other'), {
+            '19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67': 'bob@disposlab',
+        })
+        GnuPG.delete_key('test/keyhome.other', 'bob@disposlab')
+        self.assertDictEqual(GnuPG.public_keys('test/keyhome.other'), { })
+
+    def _load(self, filename):
+        with open(filename) as f:
+            return f.read()
+
+    def test_extract_fingerprint(self):
+        sample_in = '''fpr:::::::::1CD245308F0963D038E88357973CF4D9387C44D7:'''
+        fpr = GnuPG._extract_fingerprint(sample_in)
+        self.assertEqual(fpr, '1CD245308F0963D038E88357973CF4D9387C44D7')
+
+    def test_parse_uid_line(self):
+        sample_in = '''uid:e::::1624794010::C16E259AA1435947C6385B8160BC020B6C05EE18::alice@disposlab::::::::::0:'''
+        uid = GnuPG._parse_uid_line(sample_in)
+        self.assertEqual(uid, 'alice@disposlab')
+
+    def test_parse_statusfd_key_expired(self):
+        key_expired = b"""
+[GNUPG:] KEYEXPIRED 1668272263
+[GNUPG:] KEY_CONSIDERED XXXXXXXXXXXXX 0
+[GNUPG:] INV_RECP 0 name@domain
+[GNUPG:] FAILURE encrypt 1
+"""
+
+        result = GnuPG.parse_status(key_expired)
+        self.assertEqual(result['issue'], 'key expired')
+        self.assertEqual(result['recipient'], b'name@domain')
+        self.assertEqual(result['cause'], 'Unknown')
+        self.assertEqual(result['key'], b'XXXXXXXXXXXXX')
+
+    def test_parse_statusfd_key_absent(self):
+        non_specific_errors = b"""
+[GNUPG:] INV_RECP 0 name@domain
+[GNUPG:] FAILURE encrypt 1
+"""
+
+        result = GnuPG.parse_status(non_specific_errors)
+        self.assertEqual(result['issue'], b'n/a')
+        self.assertEqual(result['recipient'], b'name@domain')
+        self.assertEqual(result['cause'], 'Unknown')
+        self.assertEqual(result['key'], b'n/a')
 
 
 if __name__ == '__main__':
-	unittest.main()
+    unittest.main()

test/modules/test_lacre_core.py

@@ -0,0 +1,56 @@
+import lacre.core
+from email.message import EmailMessage
+
+import unittest
+
+
+class LacreCoreTest(unittest.TestCase):
+    def test_attachment_handling(self):
+        m = EmailMessage()
+        m.set_payload('This is a payload')
+        m.set_param('attachment', '', 'Content-Disposition')
+        m.set_param('filename', 'foo', 'Content-Disposition')
+
+        lacre.core._append_gpg_extension(m)
+
+        self.assertEqual(m.get_filename(), 'foo.pgp')
+
+    def test_attachment_handling_2(self):
+        m = EmailMessage()
+        m.set_payload('This is a payload')
+        m.set_param('attachment', '', 'Content-Disposition')
+        m.set_param('name', 'quux', 'Content-Type')
+
+        lacre.core._append_gpg_extension(m)
+
+        self.assertEqual(m.get_filename(), 'quux.pgp')
+
+    def test_payload_wrapping(self):
+        m = EmailMessage()
+        m.set_payload('This is a payload.\r\n'
+                      + '\r\n'
+                      + 'It has two paragraphs.\r\n')
+        m['Subject'] = 'Source message'
+        m.set_type('text/plain')
+        m.set_param('charset', 'utf-8')
+
+        rewrapped = lacre.core._rewrap_payload(m)
+
+        self.assertFalse('Subject' in rewrapped,
+                         'only content and content-type should be copied')
+        self.assertEqual(rewrapped.get_content_type(), 'text/plain',
+                         'rewrapped part should have initial message\'s content-type')
+
+    def test_payload_wrapping_wo_content_type(self):
+        m = EmailMessage()
+        m.set_payload('This is a payload.\r\n'
+                      + '\r\n'
+                      + 'It has two paragraphs.\r\n')
+        m['Subject'] = 'Source message'
+
+        rewrapped = lacre.core._rewrap_payload(m)
+
+        self.assertFalse('Subject' in rewrapped,
+                         'only content and content-type should be copied')
+        self.assertEqual(rewrapped.get_content_type(), 'text/plain',
+                         'rewrapped part should have initial message\'s content-type')

test/modules/test_lacre_recipients.py

@@ -0,0 +1,17 @@
+import lacre.recipients
+
+import unittest
+
+
+class RecipientListTest(unittest.TestCase):
+    def test_addition(self):
+        a_list = lacre.recipients.RecipientList()
+        a_list += lacre.recipients.GpgRecipient(
+            'alice@disposlab',
+            '1CD245308F0963D038E88357973CF4D9387C44D7')
+
+        emails = [x for x in a_list.emails()]
+        keys = [x for x in a_list.keys()]
+
+        self.assertSequenceEqual(emails, ['alice@disposlab'])
+        self.assertSequenceEqual(keys, ['1CD245308F0963D038E88357973CF4D9387C44D7'])

test/modules/test_lacre_repositories.py

@@ -0,0 +1,24 @@
+"""Lacre identity and key repository tests."""
+
+import unittest
+
+import lacre.config as conf
+import lacre.repositories as r
+import lacre.dbschema as s
+
+def ignore_sql(sql, *args, **kwargs):
+    pass
+
+class IdentityRepositoryTest(unittest.TestCase):
+
+    def setUpClass():
+        # required for init_engine to work
+        conf.load_config()
+
+    def test_freeze_identities(self):
+        eng = r.init_engine('sqlite:///test/lacre.db')
+
+        ir = r.IdentityRepository(engine=eng)
+        identities = ir.freeze_identities()
+
+        self.assertTrue(identities)

test/modules/test_lacre_text.py

@@ -1,5 +1,8 @@
 import lacre.text
 import sys
+from email import message_from_binary_file
+from email.message import EmailMessage
+from email.policy import SMTPUTF8
 
 import unittest
 
@@ -35,3 +38,23 @@ class LacreTextTest(unittest.TestCase):
         (addr2, topic) = lacre.text.parse_delimiter(addr)
         self.assertEqual(addr2, "Some.Name@example.com")
         self.assertEqual(topic, "some-topic")
+
+    def test_pgp_inline_recognised(self):
+        msg = None
+        with open('test/msgin/ed2ed.msg', 'rb') as f:
+            msg = message_from_binary_file(f, policy=SMTPUTF8)
+
+        body = msg.get_payload()
+
+        self.assertIn(lacre.text.PGP_BEGIN_S, body)
+        self.assertIn(lacre.text.PGP_END_S, body)
+        self.assertTrue(lacre.text.is_payload_pgp_inline(body))
+
+    def test_pgp_marker_mentioned(self):
+        msg = None
+        with open('test/msgin/with-markers2clear.msg', 'rb') as f:
+            msg = message_from_binary_file(f, policy=SMTPUTF8)
+
+        body = msg.get_payload()
+
+        self.assertFalse(lacre.text.is_payload_pgp_inline(body))

test/modules/test_stats.py

@@ -0,0 +1,38 @@
+import unittest
+
+from logging import getLogger, ERROR, Handler
+from lacre.stats import time_logger
+
+def make_exception_raiser(logger):
+    def f():
+        with time_logger('Just a test', logger):
+            logger.info('Doing something')
+            raise Exception('this is a test')
+    return f
+
+
+class LogRecordCollector(Handler):
+    logged_records = []
+
+    def handle(self, r):
+        self.logged_records.append(self.format(r))
+
+
+class ExecutionTimeLoggerTest(unittest.TestCase):
+    def test_exception_handling(self):
+        handler = LogRecordCollector()
+        logger = getLogger('test-logger')
+        logger.addHandler(handler)
+
+        f = make_exception_raiser(logger)
+
+        self.assertRaises(Exception, f)
+        self.assertLogs(logger, ERROR)
+        self.assertEqual(len(handler.logged_records), 3)
+        self.assertEqual(handler.logged_records[0], 'Start: Just a test')
+        self.assertEqual(handler.logged_records[1], 'Doing something')
+
+        # Exception record should include the timing result and the traceback...
+        self.assertRegex(handler.logged_records[2], '^Just a test took \\d ms, raised exception\nTraceback.*')
+        # ...as well as the original exception
+        self.assertRegex(handler.logged_records[2], 'Exception: this is a test$')

test/msgin/clear2clear.msg

@@ -1,5 +1,5 @@
-From: Dave <dave@localhost>
-To: Carlos <carlos@localhost>
-Subject: Test
-
-Body of the message.
+From: Dave <dave@localhost>
+To: Carlos <carlos@localhost>
+Subject: Test
+
+Body of the message.

test/msgin/clear2ed-delim.msg

@@ -1,5 +1,5 @@
-From: Dave <dave@localhost>
-To: Bob <bob+foobar@localhost>
-Subject: Test
-
-Body of the message.
+From: Dave <dave@localhost>
+To: Bob <bob+foobar@localhost>
+Subject: Test
+
+Body of the message.

test/msgin/clear2ed.msg

@@ -1,5 +1,5 @@
-From: Dave <dave@localhost>
-To: Bob <bob@localhost>
-Subject: Test
-
-Body of the message.
+From: Dave <dave@localhost>
+To: Bob <bob@localhost>
+Subject: Test
+
+Body of the message.

test/msgin/clear2rsa.msg

@@ -1,5 +1,5 @@
-From: Dave <dave@localhost>
-To: Alice <alice@localhost>
-Subject: Test
-
-Body of the message.
+From: Dave <dave@localhost>
+To: Alice <alice@localhost>
+Subject: Test
+
+Body of the message.

test/msgin/clear2rsa2.msg

@@ -1,6 +1,6 @@
-From: Dave <dave@disposlab
-To: Evan <evan@disposlab>
-Subject: Test
-Content-Type: text/plain; charset="utf-8"
-
-Body of the message.
+From: Dave <dave@disposlab
+To: Evan <evan@disposlab>
+Subject: Test
+Content-Type: text/plain; charset="utf-8"
+
+Body of the message.

test/msgin/ed2ed.msg

@@ -1,13 +1,13 @@
-From: Dave <dave@localhost>
-To: Bob <bob@localhost>
-Subject: Test
-Content-Transfer-Encoding: 7bit
-
------BEGIN PGP MESSAGE-----
-
-hF4DujWCoRS24dYSAQdAyGDF9Us11JDr8+XPmvlJHsMS7A4UBIcCiresJyZpSxYw
-Cqcugy5AX5fgSAiL1Cd2b1zpQ/rYdTWkFYMVbH4jBEoPC3z/aSd+hTnneJFDUdXl
-0koBDIw7NQylu6SrW+Y/DmXgalIHtwACuKivJTq/z9jdwFScV7adRR/VO53Inah3
-L1+Ho7Zta95AYW3UPu71Gw3rrkfjY4uGDiFAFg==
-=yTzD
------END PGP MESSAGE-----
+From: Dave <dave@localhost>
+To: Bob <bob@localhost>
+Subject: Test
+Content-Transfer-Encoding: 7bit
+
+-----BEGIN PGP MESSAGE-----
+
+hF4DujWCoRS24dYSAQdAyGDF9Us11JDr8+XPmvlJHsMS7A4UBIcCiresJyZpSxYw
+Cqcugy5AX5fgSAiL1Cd2b1zpQ/rYdTWkFYMVbH4jBEoPC3z/aSd+hTnneJFDUdXl
+0koBDIw7NQylu6SrW+Y/DmXgalIHtwACuKivJTq/z9jdwFScV7adRR/VO53Inah3
+L1+Ho7Zta95AYW3UPu71Gw3rrkfjY4uGDiFAFg==
+=yTzD
+-----END PGP MESSAGE-----

test/msgin/emoji.msg

@@ -0,0 +1,18 @@
+Date: Sun, 16 Apr 2023 07:29:45 +0200
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; FreeBSD amd64; rv:102.0) Gecko/20100101
+ Thunderbird/102.9.0
+Content-Language: pl
+To: Carlos <carlos@localhost>
+From: Dave <dave@localhost>
+Subject: Emoji test
+Content-Type: text/html; charset=UTF-8; format=flowed
+Content-Transfer-Encoding: 8bit
+
+<html>
+<head>
+</head>
+<body>
+àèéòìù ø Ø 🙂️ 👍️ 🚗️
+</body>
+</html>

test/msgin/html-ascii.msg

@@ -0,0 +1,18 @@
+Date: Sun, 16 Apr 2023 07:29:45 +0200
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; FreeBSD amd64; rv:102.0) Gecko/20100101
+ Thunderbird/102.9.0
+Content-Language: pl
+To: Carlos <carlos@localhost>
+From: Dave <dave@localhost>
+Subject: HTML test
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+<html>
+<head>
+</head>
+<body>
+This is just an HTML email.
+</body>
+</html>

test/msgin/html-utf8.msg

@@ -0,0 +1,19 @@
+Date: Sun, 16 Apr 2023 07:29:45 +0200
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; FreeBSD amd64; rv:102.0) Gecko/20100101
+ Thunderbird/102.9.0
+Content-Language: pl
+To: Carlos <carlos@localhost>
+From: Dave <dave@localhost>
+Subject: HTML test
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+<html>
+<head>
+</head>
+<body>
+ZAŻÓŁĆ GĘŚLĄ JAŹŃ.<br>
+zażółć gęślą jaźń.
+</body>
+</html>

test/msgin/multipart2rsa.msg

@@ -1,43 +1,43 @@
-Date: Sun, 18 Jul 2021 16:53:45 +0200
-From: User Alice <alice@disposlab>
-To: User Bob <bob@disposlab>
-Subject: encrypted
-Message-ID: <YPRAeEEc3z2M9BCy@disposlab>
-MIME-Version: 1.0
-Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
-        boundary="95hZs/zeBetwhuEy"
-Content-Disposition: inline
-Status: RO
-Content-Length: 1140
-Lines: 30
-
-
---95hZs/zeBetwhuEy
-Content-Type: application/pgp-encrypted
-Content-Disposition: attachment
-
-Version: 1
-
---95hZs/zeBetwhuEy
-Content-Type: application/octet-stream
-Content-Disposition: attachment; filename="msg.asc"
-
------BEGIN PGP MESSAGE-----
-
-hQGMA/vsqjpkmZurAQwAnb+2kDPgVFWVLkafuzVJGqFWKNtdVsvk7I1zhzFw5Hsr
-h4irSHcH0X0QjaHprNiMBDfIZaCx5VVsvGYLiu/iQkdVPXItugTpln8aAvDt8/Bp
-Hse69tgG5S9o4fPK4K2bMjNdomclDdz51cu9NXYjk/6OtzVwcSypyEmxgw24Oo1+
-Q8KfZN9n6VTXGNlrV9KnAZYs/5aaSABTeC+cDvOcjDbPAmwDHYS3qsbITYoGHnEz
-QfPIakYWPtPWkajhm4Z/iyEUSTeqew1/gAJ8sZnJpV0eg1Cr/44XgklZKFr8aJgk
-SG8PkQxsyzAZklpwMSWdbb+t9a5nEKvky3zMpdmS1GE7ubTO7nQ1geUdBiv1UUNh
-BY9d4nlGirqxX1MZUTGZidJgCy0365xbJSKkU0yFFW2uWtCKzJTEQBk3YZkNmnGH
-h8BiVvMhQ8SxKBRPeH6Zb6HHlbcgkPvJAAI4VLqkZPCBvp9irmcdFGmrgCWLxzgk
-sIjYGLA+ZuSXOKuAssXE0sAbASPAkUJRTIjzXFrCnr/MB3ZonESH01fsbsX+E/Qi
-+2oLrgjjPHcPq76fvdO6fJP6c1pM8TlOoZKn/RkPm1llULtOn4n5JZJjeUA0F2ID
-Te/U9i4YtcFZbuvw2bjeu8sAf77U6O3iTTBWkPWQT3H4YMskQc7lS1Mug6A9HL/n
-TQvAwh2MIveYyEy/y/dKeFUbpSKxyOInhTg1XtYFiT8bzEF7OEJLU9GyF5oMs67d
-o12uYlEnPhWz9oZp11aSdnyeADpVu6BQsPbwfTifcpajQSarH5sG8+rDSPju
-=7CnH
------END PGP MESSAGE-----
-
---95hZs/zeBetwhuEy--
+Date: Sun, 18 Jul 2021 16:53:45 +0200
+From: User Alice <alice@disposlab>
+To: User Bob <bob@disposlab>
+Subject: encrypted
+Message-ID: <YPRAeEEc3z2M9BCy@disposlab>
+MIME-Version: 1.0
+Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
+        boundary="95hZs/zeBetwhuEy"
+Content-Disposition: inline
+Status: RO
+Content-Length: 1140
+Lines: 30
+
+
+--95hZs/zeBetwhuEy
+Content-Type: application/pgp-encrypted
+Content-Disposition: attachment
+
+Version: 1
+
+--95hZs/zeBetwhuEy
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="msg.asc"
+
+-----BEGIN PGP MESSAGE-----
+
+hQGMA/vsqjpkmZurAQwAnb+2kDPgVFWVLkafuzVJGqFWKNtdVsvk7I1zhzFw5Hsr
+h4irSHcH0X0QjaHprNiMBDfIZaCx5VVsvGYLiu/iQkdVPXItugTpln8aAvDt8/Bp
+Hse69tgG5S9o4fPK4K2bMjNdomclDdz51cu9NXYjk/6OtzVwcSypyEmxgw24Oo1+
+Q8KfZN9n6VTXGNlrV9KnAZYs/5aaSABTeC+cDvOcjDbPAmwDHYS3qsbITYoGHnEz
+QfPIakYWPtPWkajhm4Z/iyEUSTeqew1/gAJ8sZnJpV0eg1Cr/44XgklZKFr8aJgk
+SG8PkQxsyzAZklpwMSWdbb+t9a5nEKvky3zMpdmS1GE7ubTO7nQ1geUdBiv1UUNh
+BY9d4nlGirqxX1MZUTGZidJgCy0365xbJSKkU0yFFW2uWtCKzJTEQBk3YZkNmnGH
+h8BiVvMhQ8SxKBRPeH6Zb6HHlbcgkPvJAAI4VLqkZPCBvp9irmcdFGmrgCWLxzgk
+sIjYGLA+ZuSXOKuAssXE0sAbASPAkUJRTIjzXFrCnr/MB3ZonESH01fsbsX+E/Qi
++2oLrgjjPHcPq76fvdO6fJP6c1pM8TlOoZKn/RkPm1llULtOn4n5JZJjeUA0F2ID
+Te/U9i4YtcFZbuvw2bjeu8sAf77U6O3iTTBWkPWQT3H4YMskQc7lS1Mug6A9HL/n
+TQvAwh2MIveYyEy/y/dKeFUbpSKxyOInhTg1XtYFiT8bzEF7OEJLU9GyF5oMs67d
+o12uYlEnPhWz9oZp11aSdnyeADpVu6BQsPbwfTifcpajQSarH5sG8+rDSPju
+=7CnH
+-----END PGP MESSAGE-----
+
+--95hZs/zeBetwhuEy--

test/msgin/nonascii.msg

@@ -0,0 +1,7 @@
+From: Dave <dave@localhost>
+To: Carlos <carlos@localhost>
+Content-Type: text/plain; charset="iso-8859-2"
+Content-Transfer-Encoding: 8bit
+Subject: Test
+
+£¡CZNO¦Æ. Za¼ó³æ gê¶l± ja¼ñ.

test/msgin/signed.msg

@@ -1,39 +1,39 @@
-Date: Sun, 18 Jul 2021 12:08:41 +0200
-From: User Alice <alice@disposlab>
-To: User Bob <bob@disposlab>
-Subject: signed
-Message-ID: <YPP9qer9j2u4qXsq@disposlab>
-MIME-Version: 1.0
-Content-Type: multipart/signed; micalg=pgp-sha256;
-        protocol="application/pgp-signature"; boundary="U/XjR71RAixRcb28"
-Content-Disposition: inline
-Status: RO
-Content-Length: 870
-Lines: 26
-
-
---U/XjR71RAixRcb28
-Content-Type: text/plain; charset=us-ascii
-Content-Disposition: inline
-
-A signed msg.
-
---U/XjR71RAixRcb28
-Content-Type: application/pgp-signature; name="signature.asc"
-
------BEGIN PGP SIGNATURE-----
-
-iQGzBAEBCAAdFiEEHNJFMI8JY9A46INXlzz02Th8RNcFAmDz/aUACgkQlzz02Th8
-RNdtOQv/ca8c51KoVq7CyPJUr54n4DEk/LlYniR0W51tL2a4rQxyF2AxqjdI8T4u
-bT1+bqPNYgegesyCLokeZKqhLVtCH+UVOTdtUq5bB1J7ALuuVTOIdR5woMBBsazV
-ETYEMzL6y2sGPW92ynriEw6B9pPnFKFPhOOZLrnMzM8CpkTfNmGoej+EdV74s0z4
-RayKu/WaZ1Dtx2Vy2YDtG36p/Y3n62bnzQJCRyPYfrmCxH5X5i5oibQwxLROCFNE
-4X3iVZLPHFg/DS9m4L7mBe0MJewGa1oPFr7t3ZfJ+24aJ/AvUv5uQIO+s6a7AcjD
-Pgw/IjeM/uZdPrzniZI2zsWEgsjRCL1fj49XWVNkTHrWCqLvkBg+suucNO2SR0/d
-ps+RP5mkJJHaSZyPpxwo9/PHKX67Mkpn/uEXlE8nV6IqKoXRzr1N0qwyhvbZQZLD
-FMumxx/eOSiOpaiRhGhoZiUpf+VdnV/1ClpAcdbthy/psx/CMYVblAM8xg74NR9+
-Q/WlFbRl
-=uMdE
------END PGP SIGNATURE-----
-
---U/XjR71RAixRcb28--
+Date: Sun, 18 Jul 2021 12:08:41 +0200
+From: User Alice <alice@disposlab>
+To: User Bob <bob@disposlab>
+Subject: signed
+Message-ID: <YPP9qer9j2u4qXsq@disposlab>
+MIME-Version: 1.0
+Content-Type: multipart/signed; micalg=pgp-sha256;
+        protocol="application/pgp-signature"; boundary="U/XjR71RAixRcb28"
+Content-Disposition: inline
+Status: RO
+Content-Length: 870
+Lines: 26
+
+
+--U/XjR71RAixRcb28
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+A signed msg.
+
+--U/XjR71RAixRcb28
+Content-Type: application/pgp-signature; name="signature.asc"
+
+-----BEGIN PGP SIGNATURE-----
+
+iQGzBAEBCAAdFiEEHNJFMI8JY9A46INXlzz02Th8RNcFAmDz/aUACgkQlzz02Th8
+RNdtOQv/ca8c51KoVq7CyPJUr54n4DEk/LlYniR0W51tL2a4rQxyF2AxqjdI8T4u
+bT1+bqPNYgegesyCLokeZKqhLVtCH+UVOTdtUq5bB1J7ALuuVTOIdR5woMBBsazV
+ETYEMzL6y2sGPW92ynriEw6B9pPnFKFPhOOZLrnMzM8CpkTfNmGoej+EdV74s0z4
+RayKu/WaZ1Dtx2Vy2YDtG36p/Y3n62bnzQJCRyPYfrmCxH5X5i5oibQwxLROCFNE
+4X3iVZLPHFg/DS9m4L7mBe0MJewGa1oPFr7t3ZfJ+24aJ/AvUv5uQIO+s6a7AcjD
+Pgw/IjeM/uZdPrzniZI2zsWEgsjRCL1fj49XWVNkTHrWCqLvkBg+suucNO2SR0/d
+ps+RP5mkJJHaSZyPpxwo9/PHKX67Mkpn/uEXlE8nV6IqKoXRzr1N0qwyhvbZQZLD
+FMumxx/eOSiOpaiRhGhoZiUpf+VdnV/1ClpAcdbthy/psx/CMYVblAM8xg74NR9+
+Q/WlFbRl
+=uMdE
+-----END PGP SIGNATURE-----
+
+--U/XjR71RAixRcb28--

test/msgin/utf8-alternative.msg

@@ -0,0 +1,43 @@
+Content-Type: multipart/alternative;
+ boundary="------------6s7R3c0y2W8qiD7cU3iWyXcw"
+Date: Wed, 23 Nov 2022 08:06:29 +0100
+MIME-Version: 1.0
+Content-Language: pl-PL
+From: Dave <dave@localhost>
+To: carlos@disposlab
+Subject: Lorem ipsum...
+
+This is a multi-part message in MIME format.
+--------------6s7R3c0y2W8qiD7cU3iWyXcw
+Content-Type: text/plain; charset=UTF-8; format=flowed
+Content-Transfer-Encoding: 8bit
+
+Современная литература - это всемирное культурное богатство, наследие 
+всех людей, которые находят вдохновение в книгах. Читать - значит жить, 
+вникать в поток мыслей других.
+
+Współczesna literatura to światowe bogactwo kulturowe, dziedzictwo 
+wszystkich ludzi, którzy znajdują inspirację w książkach. Czytać to żyć, 
+zagłębiać się w przepływ myśli innych.
+
+--------------6s7R3c0y2W8qiD7cU3iWyXcw
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+<html>
+  <head>
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+  </head>
+  <body text="#dcdcdc" bgcolor="#3b3b3b">
+    <p>Современная литература - это всемирное культурное богатство,
+      наследие всех людей, которые находят вдохновение в книгах. Читать
+      - значит жить, вникать в поток мыслей других.<br>
+      <br>
+      Współczesna literatura to światowe bogactwo kulturowe, dziedzictwo
+      wszystkich ludzi, którzy znajdują inspirację w książkach. Czytać
+      to żyć, zagłębiać się w przepływ myśli innych.<br>
+    </p>
+  </body>
+</html>
+
+--------------6s7R3c0y2W8qiD7cU3iWyXcw--

test/msgin/utf8-plain.msg

@@ -0,0 +1,29 @@
+Date: Thu, 15 Dec 2022 21:40:51 +0100
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; FreeBSD amd64; rv:102.0) Gecko/20100101
+ Thunderbird/102.5.1
+Subject: Lorem_ipsum, text/plain
+Content-Language: pl
+To: somebody@disposlab
+From: Dave <dave@localhost>
+Content-Type: text/plain; charset=UTF-8; format=flowed
+
+siema :)
+poniżej tekst, o który prosiłeś. o coś takiego chodziło? :) jeśli trzeba 
+poprawić, daj znać!
+
+pzdr!
+
+łukasz
+
+***
+
+Современная литература - это всемирное культурное богатство, наследие 
+всех людей, которые находят вдохновение в книгах. Читать - значит жить, 
+вникать в поток мыслей других.
+
+// tłumaczenie:
+
+Współczesna literatura to światowe bogactwo kulturowe, dziedzictwo 
+wszystkich ludzi, którzy znajdują inspirację w książkach. Czytać to żyć, 
+zagłębiać się w przepływ myśli innych.

test/msgin/utf8.msg

@@ -0,0 +1,7 @@
+From: Dave <dave@localhost>
+To: Carlos <carlos@localhost>
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 8bit
+Subject: Test
+
+ŁĄCZNOŚĆ. Zaźółć gęślą jaźń.

test/msgin/with-markers2clear.msg

@@ -0,0 +1,11 @@
+From: Dave <dave@localhost>
+To: Carlos <carlos@localhost>
+Subject: PGP markers
+
+This message includes inline PGP markers.
+It's enough to include these two lines:
+
+-----BEGIN PGP MESSAGE-----
+-----END PGP MESSAGE-----
+
+Test logs will give a hint which path this message takes.

test/utils/relay.py

@@ -1,7 +1,7 @@
 #!/usr/local/bin/python2
 #
 # This quick-and-dirty script supports only the happy case of SMTP session,
-# i.e. what gpg-mailgate/gpg-lacre needs to deliver encrypted email.
+# i.e. what lacre needs to deliver encrypted email.
 #
 # It listens on the port given as the only command-line argument and consumes a
 # message, then prints it to standard output.  The goal is to be able to
@@ -12,77 +12,100 @@ import sys
 import socket
 
 import logging
+import email
+import email.policy
 
 
 EXIT_UNAVAILABLE = 1
+EXIT_NETWORK = 2
+EXIT_UNKNOWN = 3
 
 ENCODING = 'utf-8'
 
 BUFFER_SIZE = 4096
-EOM = "\r\n.\r\n"
+EOM = b"\r\n.\r\n"
 LAST_LINE = -3
 
 
-def welcome(msg):
+def _welcome(msg):
     return b"220 %b\r\n" % (msg)
 
-def ok(msg = b"OK"):
+
+def _ok(msg=b"OK"):
     return b"250 %b\r\n" % (msg)
 
-def bye():
+
+def _bye():
     return b"251 Bye"
 
-def provide_message():
+
+def _provide_message():
     return b"354 Enter a message, ending it with a '.' on a line by itself\r\n"
 
-def receive_and_confirm(session):
-    session.recv(BUFFER_SIZE)
-    session.sendall(ok())
 
-def localhost_at(port):
+def _receive_and_confirm(session):
+    session.recv(BUFFER_SIZE)
+    session.sendall(_ok())
+
+
+def _receive_and_ignore(session):
+    session.recv(BUFFER_SIZE)
+
+
+def _localhost_at(port):
     return ('127.0.0.1', port)
 
-def serve(port):
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+def _receive_bytes(conn) -> bytes:
+    return conn.recv(BUFFER_SIZE)
+
+
+def _listen(port, sock):
     try:
-        s.bind(localhost_at(port))
-        logging.info(f"Listening on localhost, port {port}")
-        s.listen(1)
-        logging.info("Listening...")
+        sock.bind(_localhost_at(port))
+        sock.listen(1)
     except socket.error as e:
         print("Cannot connect", e)
-        logging.error(f"Cannot connect {e}")
-        sys.exit(EXIT_UNAVAILABLE)
+        logging.exception('Cannot connect')
+        sys.exit(EXIT_NETWORK)
+
+
+def _serve(port) -> bytes:
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    _listen(port, s)
 
     logging.debug("About to accept a connection...")
     (conn, addr) = s.accept()
     logging.debug(f"Accepting connection from {conn}")
-    conn.sendall(welcome(b"TEST SERVER"))
+    conn.sendall(_welcome(b"TEST SERVER"))
 
-    receive_and_confirm(conn)                       # Ignore HELO/EHLO
-    receive_and_confirm(conn)                       # Ignore sender address
-    receive_and_confirm(conn)                       # Ignore recipient address
+    _receive_and_confirm(conn)                       # Ignore HELO/EHLO
+    _receive_and_confirm(conn)                       # Ignore sender address
+    _receive_and_confirm(conn)                       # Ignore recipient address
 
-    conn.recv(BUFFER_SIZE)
-    conn.sendall(provide_message())
+    _receive_and_ignore(conn)
+    conn.sendall(_provide_message())
 
     # Consume until we get <CR><LF>.<CR><LF>, the end-of-message marker.
-    message = ''
+    message = b''
     while not message.endswith(EOM):
-        message += conn.recv(BUFFER_SIZE).decode(ENCODING)
-    conn.sendall(ok(b"OK, id=test"))
+        buf = _receive_bytes(conn)
+        logging.debug('Received data: %s', buf)
+        message += buf
+    conn.sendall(_ok(b"OK, id=test"))
 
     conn.recv(BUFFER_SIZE)
-    conn.sendall(bye())
+    conn.sendall(_bye())
 
     conn.close()
 
-    logging.debug(f"Received {len(message)} characters of data")
+    logging.debug('Received %d bytes of data', len(message))
 
     # Trim EOM marker as we're only interested in the message body.
     return message[:-len(EOM)]
 
-def error(msg, exit_code):
+
+def _error(msg, exit_code):
     logging.error(msg)
     print("ERROR: %s" % (msg))
     sys.exit(exit_code)
@@ -96,9 +119,18 @@ logging.basicConfig(filename='test/logs/relay.log',
                     level=logging.DEBUG)
 
 if len(sys.argv) < 2:
-    error("Usage: relay.py PORT_NUMBER", EXIT_UNAVAILABLE)
+    _error("Usage: relay.py PORT_NUMBER", EXIT_UNAVAILABLE)
 
 port = int(sys.argv[1])
-body = serve(port)
 
-print(body)
+try:
+    body = _serve(port)
+    logging.debug('Parsing message')
+    msg = email.message_from_bytes(body, policy=email.policy.SMTP)
+    print(msg)
+except ConnectionResetError:
+    logging.exception('Communication issue')
+    _error('Could not receive complete message', EXIT_NETWORK)
+except BrokenPipeError:
+    logging.exception('Pipe error')
+    _error('Pipe error', EXIT_UNKNOWN)

test/utils/schema.py

@@ -3,23 +3,27 @@ import sqlalchemy
 from sqlalchemy.sql import insert
 
 def define_db_schema():
-	meta = sqlalchemy.MetaData()
+    meta = sqlalchemy.MetaData()
 
-	gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta,
-								  sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
-								  sqlalchemy.Column('email', sqlalchemy.String(256)),
-								  sqlalchemy.Column('publickey', sqlalchemy.Text),
-								  sqlalchemy.Column('confirm', sqlalchemy.String(32)),
-								  sqlalchemy.Column('status', sqlalchemy.Integer),
-								  sqlalchemy.Column('time', sqlalchemy.DateTime))
+    lacre_keys = sqlalchemy.Table('lacre_keys', meta,
+                                  sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
+                                  sqlalchemy.Column('email', sqlalchemy.String(256)),
+                                  sqlalchemy.Column('publickey', sqlalchemy.Text),
+                                  sqlalchemy.Column('confirm', sqlalchemy.String(32)),
+                                  sqlalchemy.Column('status', sqlalchemy.Integer),
+                                  sqlalchemy.Column('time', sqlalchemy.DateTime))
 
-	return (meta, gpgmw_keys)
+    identities = sqlalchemy.Table('lacre_identities', meta,
+        sqlalchemy.Column('email', sqlalchemy.String(256), index=True),
+        sqlalchemy.Column('fingerprint', sqlalchemy.String(64), index=True))
+
+    return (meta, lacre_keys, identities)
 
 if len(sys.argv) != 2:
-	print("ERROR: output database missing")
-	sys.exit(1)
+    print("ERROR: output database missing")
+    sys.exit(1)
 
-(meta, gpgmw_keys) = define_db_schema()
+(meta, lacre_keys, identities) = define_db_schema()
 
 dbname = sys.argv[1]
 test_db = sqlalchemy.create_engine(f"sqlite:///{dbname}")
@@ -30,8 +34,8 @@ meta.create_all(test_db)
 conn = test_db.connect()
 
 # Populate the database with dummy data
-conn.execute(gpgmw_keys.insert(), [
-	{"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
+conn.execute(lacre_keys.insert(), [
+    {"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
 \n\
 mQGNBGDYY5oBDAC+HAVjA05jsIpHfQ2KQ9m2olo1Qnlk+dkjD+Gagxj1ACezyiGL\n\
 cfZfoE/MJYLCH9yPcX1fUIAPwdAyfJKlvkVcz+MhEpgl3aP3NM2L2unSx3v9ZFwT\n\
@@ -73,7 +77,7 @@ pw==\n\
 =Tbwz\n\
 -----END PGP PUBLIC KEY BLOCK-----\
 ", "status": 0, "confirm": "", "time": None},
-	{"id": 2, "email": "bob@disposlab",   "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
+    {"id": 2, "email": "bob@disposlab",   "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
 \n\
 mDMEYdTFkRYJKwYBBAHaRw8BAQdA2tgdP1pMt3cv3XAW7ov5AFn74mMZvyTksp9Q\n\
 eO1PkpK0GkJvYiBGb29iYXIgPGJvYkBkaXNwb3NsYWI+iJYEExYIAD4WIQQZz0tH\n\
@@ -86,6 +90,12 @@ AQgHiHgEGBYIACAWIQQZz0tH7MnEevqE1L2W85/aDjG7ZwUCYdTF4AIbDAAKCRCW\n\
 OjjB6xRD0Q2FN+alsNGCtdutAs18AZ5l33RMzws=\n\
 =wWoq\n\
 -----END PGP PUBLIC KEY BLOCK-----\
-", "status": 0, "confirm": "", "time": None},
-	{"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 0, "confirm": "", "time": None}
-	])
+", "status": 1, "confirm": "", "time": None},
+    {"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 2, "confirm": "", "time": None}
+    ])
+
+conn.execute(identities.insert(), [
+    {'fingerprint': '1CD245308F0963D038E88357973CF4D9387C44D7', 'email': 'alice@disposlab'},
+    {'fingerprint': '19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67', 'email': 'bob@disposlab'},
+    {'fingerprint': '530B1BB2D0CC7971648198BBA4774E507D3AF5BC', 'email': 'evan@disposlab'}
+   ])

test/utils/sendmail.py

@@ -2,27 +2,49 @@ import logging
 import smtplib
 import sys
 import getopt
+from email import message_from_binary_file
+from email.policy import SMTPUTF8
 
 
-def _load_file(name):
-    f = open(name, 'r')
-    contents = f.read()
-    f.close()
-    return contents
+def _load_file(name) -> bytes:
+    with open(name, 'rb') as f:
+        return f.read()
 
 
-def _send(host, port, from_addr, recipients, message):
+def _load_message(name):
+    with open(name, 'rb') as f:
+        return message_from_binary_file(f, policy=SMTPUTF8)
+
+
+def _send_message(host, port, from_addr, recipients, message):
     logging.info(f"From {from_addr} to {recipients} at {host}:{port}")
     try:
         smtp = smtplib.SMTP(host, port)
-        # smtp.starttls()
-        return smtp.sendmail(from_addr, recipients, message)
+        return smtp.sendmail(from_addr, recipients, message.as_bytes())
     except smtplib.SMTPDataError as e:
         logging.error(f"Couldn't deliver message. Got error: {e}")
         return None
     except ConnectionRefusedError as e:
         logging.exception(f"Connection refused: {e}")
         return None
+    except:
+        logging.exception('Unexpected exception was thrown')
+        return None
+
+
+# The poinf of this function is to do _almost_ what SMTP.sendmail does, but
+# without enforcing ASCII.  We want to test Lacre with not necessarily valid
+# messages.
+def _send_bytes(host: str, port, from_addr: str, recipients, message: bytes):
+    try:
+        smtp = smtplib.SMTP(host, port)
+        smtp.ehlo_or_helo_if_needed()
+        smtp.mail(from_addr)
+        for r in recipients:
+            smtp.rcpt(r)
+        smtp.data(message)
+    except:
+        logging.exception('Unexpected exception was thrown')
 
 
 logging.basicConfig(filename="test/logs/sendmail.log",
@@ -36,21 +58,15 @@ opts, _ = getopt.getopt(sys.argv[1:], "f:t:m:")
 for opt, value in opts:
     if opt == "-f":
         sender = value
-        logging.debug(f"Sender is {sender}")
+        logging.debug(f"Sender is {sender!r}")
     if opt == "-t":
         recipient = value
-        logging.debug(f"Recipient is {recipient}")
+        logging.debug(f"Recipient is {recipient!r}")
     if opt == "-m":
         message = _load_file(value)
         logging.debug(f"Message is {message}")
 
-if message is None:
-    message = """\
-From: dave@disposlab
-To: alice@disposlab
-Subject: Test message
+if message is None or sender is None or recipient is None:
+    print('Use options to provide: -f sender -t recipient -m message')
 
-Lorem ipsum dolor sit amet.
-"""
-
-_send('localhost', 10025, sender, [recipient], message)
+_send_bytes('localhost', 10025, sender, [recipient], message)

webgate-cron.py

@@ -1,154 +1,124 @@
 #!/usr/bin/python
 
 #
-# gpg-mailgate
+# lacre
 #
-# This file is part of the gpg-mailgate source code.
+# This file is part of the lacre source code.
 #
-# gpg-mailgate is free software: you can redistribute it and/or modify
+# lacre is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# gpg-mailgate source code is distributed in the hope that it will be useful,
+# lacre source code is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with gpg-mailgate source code. If not, see <http://www.gnu.org/licenses/>.
+# along with lacre source code. If not, see <http://www.gnu.org/licenses/>.
 #
 
-import GnuPG
-import sqlalchemy
-from sqlalchemy.sql import select, delete, and_
-import smtplib
-import markdown
-from email.mime.text import MIMEText
-from email.mime.multipart import MIMEMultipart
-
+import sys
 import logging
 import lacre
 import lacre.config as conf
+from lacre.notify import notify
 
-
-def _load_file(name):
-    f = open(name)
-    data = f.read()
-    f.close()
-    return data
-
-
-def _authenticate_maybe(smtp):
-    if conf.config_item_equals('smtp', 'enabled', 'true'):
-        LOG.debug(f"Connecting to {conf.get_item('smtp', 'host')}:{conf.get_item('smtp', 'port')}")
-        smtp.connect(conf.get_item('smtp', 'host'), conf.get_item('smtp', 'port'))
-        smtp.ehlo()
-        if conf.config_item_equals('smtp', 'starttls', 'true'):
-            LOG.debug("StartTLS enabled")
-            smtp.starttls()
-        smtp.ehlo()
-        smtp.login(conf.get_item('smtp', 'username'), conf.get_item('smtp', 'password'))
-
-
-def _send_msg(mailsubject, messagefile, recipients = None):
-    mailbody = _load_file(conf.get_item('cron', 'mail_templates') + "/" + messagefile)
-    msg = MIMEMultipart("alternative")
-
-    msg["From"] = conf.get_item('cron', 'notification_email')
-    msg["To"] = recipients
-    msg["Subject"] = mailsubject
-
-    msg.attach(MIMEText(mailbody, 'plain'))
-    msg.attach(MIMEText(markdown.markdown(mailbody), 'html'))
-
-    if conf.config_item_set('relay', 'host') and conf.config_item_set('relay', 'enc_port'):
-        relay = (conf.get_item('relay', 'host'), int(conf.get_item('relay', 'enc_port')))
-        smtp = smtplib.SMTP(relay[0], relay[1])
-        _authenticate_maybe(smtp)
-        smtp.sendmail(conf.get_item('cron', 'notification_email'), recipients, msg.as_string())
-    else:
-        LOG.info("Could not send mail due to wrong configuration")
-
-
-def _setup_db_connection(url):
-    engine = sqlalchemy.create_engine(url)
-    return (engine, engine.connect())
-
-
-def _define_db_schema():
-    meta = sqlalchemy.MetaData()
-
-    gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta,
-                                  sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
-                                  sqlalchemy.Column('email', sqlalchemy.String(256)),
-                                  sqlalchemy.Column('publickey', sqlalchemy.Text),
-                                  sqlalchemy.Column('confirm', sqlalchemy.String(32)),
-                                  sqlalchemy.Column('status', sqlalchemy.Integer),
-                                  sqlalchemy.Column('time', sqlalchemy.DateTime))
-
-    return (gpgmw_keys)
-
-
-# Read configuration from /etc/gpg-mailgate.conf
+# Read configuration from /etc/lacre.conf
 conf.load_config()
 
 lacre.init_logging(conf.get_item('logging', 'config'))
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger('webgate-cron.py')
+
+import GnuPG
+from lacre.repositories import KeyConfirmationQueue, IdentityRepository, init_engine
 
 
-if conf.config_item_equals('database', 'enabled', 'yes') and conf.config_item_set('database', 'url'):
-    (engine, conn) = _setup_db_connection(conf.get_item("database", "url"))
-    (gpgmw_keys) = _define_db_schema()
+def _validate_config():
+    missing = conf.validate_config(additional=conf.CRON_REQUIRED)
+    if missing:
+        LOG.error('Missing config parameters: %s', missing)
+        exit(lacre.EX_CONFIG)
 
-    selq = select(gpgmw_keys.c.publickey, gpgmw_keys.c.id, gpgmw_keys.c.email)\
-        .where(and_(gpgmw_keys.c.status == 0, gpgmw_keys.c.confirm == ""))\
-        .limit(100)
-    LOG.debug(f"Retrieving keys to be processed: {selq}")
-    result_set = conn.execute(selq)
 
-    for row in result_set:
-        # delete any other public keys associated with this confirmed email address
-        delq = delete(gpgmw_keys).where(and_(gpgmw_keys.c.email == row[2], gpgmw_keys.c.id != row[1]))
-        LOG.debug(f"Deleting public keys associated with confirmed email: {delq}")
-        conn.execute(delq)
-        GnuPG.delete_key(conf.get_item('gpg', 'keyhome'), row[2])
-        LOG.info('Deleted key for <' + row[2] + '> via import request')
+def import_key(key_dir, armored_key, key_id, email, key_queue, identities):
+    # import the key to gpg
+    (fingerprint, _) = GnuPG.add_key(key_dir, armored_key)
 
-        if row[0].strip():  # we have this so that user can submit blank key to remove any encryption
-            if GnuPG.confirm_key(row[0], row[2]):
-                GnuPG.add_key(conf.get_item('gpg', 'keyhome'), row[0])  # import the key to gpg
-                modq = gpgmw_keys.update().where(gpgmw_keys.c.id == row[1]).values(status=1)
-                LOG.debug(f"Key imported, updating key: {modq}")
-                conn.execute(modq)                      # mark key as accepted
-                LOG.warning('Imported key from <' + row[2] + '>')
-                if conf.config_item_equals('cron', 'send_email', 'yes'):
-                    _send_msg("PGP key registration successful", "registrationSuccess.md", row[2])
-            else:
-                delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1])
-                LOG.debug(f"Cannot confirm key, deleting it: {delq}")
-                conn.execute(delq)              # delete key
-                LOG.warning('Import confirmation failed for <' + row[2] + '>')
-                if conf.config_item_equals('cron', 'send_email', 'yes'):
-                    _send_msg("PGP key registration failed", "registrationError.md", row[2])
-        else:
-            # delete key so we don't continue processing it
-            delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1])
-            LOG.debug(f"Deleting key: {delq}")
-            conn.execute(delq)
-            if conf.config_item_equals('cron', 'send_email', 'yes'):
-                _send_msg("PGP key deleted", "keyDeleted.md", row[2])
+    key_queue.mark_accepted(key_id)
+    identities.register_or_update(email, fingerprint)
 
-    # delete keys
-    stat2q = select(gpgmw_keys.c.email, gpgmw_keys.c.id).where(gpgmw_keys.c.status == 2).limit(100)
-    stat2_result_set = conn.execute(stat2q)
+    LOG.info('Imported key from: %s', email)
+    if conf.flag_enabled('cron', 'send_email'):
+        notify("PGP key registration successful", "registrationSuccess.md", email)
 
-    for row in stat2_result_set:
-        GnuPG.delete_key(conf.get_item('gpg', 'keyhome'), row[0])
-        delq = delete(gpgmw_keys).where(gpgmw_keys.c.id == row[1])
-        LOG.debug(f"Deleting keys that have already been processed: {delq}")
-        conn.execute(delq)
-        LOG.info('Deleted key for <' + row[0] + '>')
-else:
+
+def import_failed(key_id, email, key_queue):
+    key_queue.delete_keys(key_id)
+    LOG.warning('Import confirmation failed: %s', email)
+
+    if conf.flag_enabled('cron', 'send_email'):
+        notify("PGP key registration failed", "registrationError.md", email)
+
+
+def delete_key(key_id, email, key_queue):
+    # delete key so we don't continue processing it
+    LOG.debug('Empty key received, just deleting')
+
+    key_queue.delete_keys(row_id)
+    if conf.flag_enabled('cron', 'send_email'):
+        notify("PGP key deleted", "keyDeleted.md", email)
+
+
+def cleanup(key_dir, key_queue):
+    """Delete keys and queue entries."""
+
+    LOG.info('Cleaning up after a round of key confirmation')
+    for email, row_id in key_queue.fetch_keys_to_delete():
+        LOG.debug('Removing key from keyring: %s', email)
+        GnuPG.delete_key(key_dir, email)
+
+        LOG.debug('Removing key from identity store: %s', row_id)
+        key_queue.delete_keys(row_id)
+
+        LOG.info('Deleted key for: %s', email)
+
+
+_validate_config()
+
+if not (conf.flag_enabled('database', 'enabled') and conf.config_item_set('database', 'url')):
     print("Warning: doing nothing since database settings are not configured!")
     LOG.error("Warning: doing nothing since database settings are not configured!")
+    sys.exit(lacre.EX_CONFIG)
+
+
+try:
+    db_engine = init_engine(conf.get_item('database', 'url'))
+
+    identities = IdentityRepository(engine=db_engine)
+    key_queue = KeyConfirmationQueue(engine=db_engine)
+
+    key_dir = conf.get_item('gpg', 'keyhome')
+    LOG.debug('Using GnuPG with home directory in %s', key_dir)
+
+    for armored_key, row_id, email in key_queue.fetch_keys():
+        # delete any other public keys associated with this confirmed email address
+        key_queue.delete_keys(row_id, email=email)
+        identities.delete(email)
+        GnuPG.delete_key(key_dir, email)
+        LOG.info('Deleted key via import request for: %s', email)
+
+        if not armored_key.strip():  # we have this so that user can submit blank key to remove any encryption
+            # delete key so we don't continue processing it
+            delete_key(row_id, email, key_queue)
+            continue
+
+        if GnuPG.confirm_key(armored_key, email):
+            import_key(key_dir, armored_key, row_id, email, key_queue, identities)
+        else:
+            import_failed(row_id, email, key_queue)
+
+    cleanup(key_dir, key_queue)
+except:
+    LOG.exception('Unexpected issue during key confirmation')