Disroot/gpg-lacre
8
15
Fork 4
Code Issues 37 Pull requests 1 Releases 3 Wiki Activity

Add basic support for RDBMS-based keyring

Browse source
This commit is contained in:
Piotr F. Mieszkowski 2023-09-30 22:38:33 +02:00
parent 274bfbaf3b
commit 41442e5b59
9 changed files with 335 additions and 194 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
gpg-mailgate.conf.sample
View file

@ -27,6 +27,19 @@ mail_case_insensitive = no
# i.e. have mode 700.
keyhome = /var/gpgmailgate/.gnupg
[keyring]
# Two options available:
#
# - file -- use GnuPG's pubring.kbx file as the only key store and cache its
# contents, reloading it on file modifications.
#
# - database -- use a relational database to store identities (when used, you
# must specify url parameter to specify which database to connect to).
type = file
# Only required when type==DatabaseKeyRing, specifies the database URL.
#url = file:///path/to/sqlite.db
[smime]
# the directory for the S/MIME certificate files
cert_path = /var/gpgmailgate/smime

57
lacre/_keyringcommon.py Normal file
View file

@ -0,0 +1,57 @@
class KeyCache:
"""A store for OpenPGP keys.
Key case is sanitised while loading from GnuPG if so
configured. See mail_case_insensitive parameter in section
[default].
"""
def __init__(self, keys: dict = None):
"""Initialise an empty cache.
With keyring_dir given, set location of the directory from which keys should be loaded.
"""
self._keys = keys
def __getitem__(self, fingerpring):
"""Look up email assigned to the given fingerprint."""
return self._keys[fingerpring]
def __setitem__(self, fingerprint, email):
"""Assign an email to a fingerpring, overwriting it if it was already present."""
self._keys[fingerprint] = email
def __contains__(self, fingerprint):
"""Check if the given fingerprint is assigned to an email."""
# This method has to be present for KeyCache to be a dict substitute.
# See mailgate, function _identify_gpg_recipients.
return fingerprint in self._keys
def has_email(self, email):
"""Check if cache contains a key assigned to the given email."""
return email in self._keys.values()
def __repr__(self):
"""Return text representation of this object."""
details = ' '.join(self._keys.keys())
return '<KeyCache %s>' % (details)
class KeyRing:
"""Contract to be implemented by a key-store (a.k.a. keyring)."""
def load(self):
"""Load keyring, replacing any previous contents of the cache."""
raise NotImplementedError('KeyRing.load not implemented')
async def freeze_identities(self) -> KeyCache:
"""Return a static, async-safe copy of the identity map."""
raise NotImplementedError('KeyRing.load not implemented')
def post_init_hook(self):
"""Lets the keyring perform additional operations following its initialisation."""
pass
def shutdown(self):
"""Lets the keyring perform operations prior to shutting down."""
pass

21
lacre/daemon.py
View file

@ -11,7 +11,6 @@ import asyncio
import email
from email.policy import SMTPUTF8
import time
from watchdog.observers import Observer
# Load configuration and init logging, in this order. Only then can we load
# the last Lacre module, i.e. lacre.mailgate.
@ -106,13 +105,6 @@ def _init_controller(keys: kcache.KeyRing, max_body_bytes=None, tout: float = 5)
data_size_limit=max_body_bytes)
def _init_reloader(keyring_dir: str, reloader) -> kcache.KeyringModificationListener:
listener = kcache.KeyringModificationListener(reloader)
observer = Observer()
observer.schedule(listener, keyring_dir, recursive=False)
return observer
def _validate_config():
missing = conf.validate_config()
if missing:
@ -138,12 +130,12 @@ def _main():
loop = asyncio.get_event_loop()
keyring = kcache.KeyRing(keyring_path, loop)
controller = _init_controller(keyring, max_data_bytes)
reloader = _init_reloader(keyring_path, keyring)
mode = conf.get_item('keyring', 'type')
LOG.info(f'Watching keyring directory {keyring_path}...')
reloader.start()
keyring = kcache.init_keyring(mode, loop = loop)
controller = _init_controller(keyring, max_data_bytes)
keyring.post_init_hook()
LOG.info('Starting the daemon...')
controller.start()
@ -156,8 +148,7 @@ def _main():
LOG.exception('Unexpected exception caught, your system may be unstable')
finally:
LOG.info('Shutting down keyring watcher and the daemon...')
reloader.stop()
reloader.join()
keyring.shutdown()
controller.stop()
LOG.info("Done")

47
lacre/dbkeyring.py Normal file
View file

@ -0,0 +1,47 @@
"""Database-backed keyring implementation."""
from lacre._keyringcommon import KeyRing, KeyCache
import logging
import sqlalchemy
from sqlalchemy.sql import select
LOG = logging.getLogger(__name__)
class KeyRingSchema:
def __init__(self):
self._meta = sqlalchemy.MetaData()
self._id_table = self._identities()
def _identities(self):
lacre_id = sqlalchemy.Table('identities', self._meta,
sqlalchemy.Column('email', sqlalchemy.String(256), index=True),
sqlalchemy.Column('key_id', sqlalchemy.String(64), index=True))
return lacre_id
def identities(self):
return self._id_table
class DatabaseKeyRing(KeyRing):
"""Database-backed key storage."""
def __init__(self, database_url, schema: KeyRingSchema):
self._schema = schema
self._engine = sqlalchemy.create_engine(database_url)
self._connection = self._engine.connect()
def load(self):
"""Do nothing, database contents doesn't need to be cached."""
pass
async def freeze_identities(self) -> KeyCache:
"""Return a static, async-safe copy of the identity map."""
return self._load_identities()
def _load_identities(self) -> KeyCache:
identities = self._schema.identities()
all_identities = select(identities.c.key_id, identities.c.email)
result = self._connection.execute(all_identities)
LOG.debug('Retrieving all keys')
return KeyCache({key_id: email for key_id, email in result})

149
lacre/filekeyring.py Normal file
View file

@ -0,0 +1,149 @@
"""File-based keyring.
It's a wrapper over GnuPG module that just executes gpg command.
'"""
import logging
import GnuPG
import copy
from os import stat
from watchdog.events import FileSystemEventHandler, FileSystemEvent
from watchdog.observers import Observer
from asyncio import Semaphore, create_task, get_event_loop, run
import lacre.text as text
import lacre.config as conf
from lacre._keyringcommon import KeyRing, KeyCache
LOG = logging.getLogger(__name__)
def _sanitize(keys):
sanitize = text.choose_sanitizer(conf.get_item('default', 'mail_case_insensitive'))
return {fingerprint: sanitize(keys[fingerprint]) for fingerprint in keys}
class FileKeyRing(KeyRing):
"""A high-level adapter for GnuPG-maintained keyring directory.
Its role is to keep a cache of keys present in the keyring,
reload it when necessary and produce static copies of
fingerprint=>email maps.
"""
def __init__(self, path: str, loop=None):
"""Initialise the adapter."""
self._path = path
self._keys = self._load_and_sanitize()
self._sema = Semaphore()
self._last_mod = None
self._loop = loop or get_event_loop()
def _load_and_sanitize(self):
keys = self._load_keyring_from(self._path)
return _sanitize(keys)
def _load_keyring_from(self, keyring_dir):
return GnuPG.public_keys(keyring_dir)
async def freeze_identities(self) -> KeyCache:
"""Return a static, async-safe copy of the identity map."""
async with self._sema:
keys = copy.deepcopy(self._keys)
return KeyCache(keys)
def load(self):
"""Load keyring, replacing any previous contents of the cache."""
LOG.debug('Reloading keys...')
tsk = create_task(self._load(), 'LoadTask')
self._loop.run_until_complete(tsk)
async def _load(self):
last_mod = self._read_mod_time()
LOG.debug(f'Keyring was last modified: {last_mod}')
if self._is_modified(last_mod):
LOG.debug('Keyring has been modified')
async with self._sema:
LOG.debug('About to re-load the keyring')
self.replace_keyring(self._load_keyring_from(self._path))
else:
LOG.debug('Keyring not modified recently, continuing')
self._last_mod = self._read_mod_time()
reload = load
def replace_keyring(self, keys: dict):
"""Overwrite previously stored key cache with KEYS."""
keys = _sanitize(keys)
LOG.info(f'Storing {len(keys)} keys')
self._keys = keys
def _read_mod_time(self) -> int:
# (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)
# 0 1 2 3 4 5 6 7 8 9
MTIME = 8
st = stat(self._path)
return st[MTIME]
def _is_modified(self, last_mod):
if self._last_mod is None:
LOG.debug('Keyring not loaded before')
return True
elif self._last_mod != last_mod:
LOG.debug('Keyring directory mtime changed')
return True
else:
LOG.debug('Keyring not modified ')
return False
def __repr__(self) -> str:
"""Return text representation of this keyring."""
return '<KeyRing path=%s last_mod=%d>' % (self._path, self._last_mod)
def post_init_hook(self):
self._reloader = init_reloader(self._path, self)
LOG.info(f'Watching keyring directory {self._path}...')
self._reloader.start()
def shutdown(self):
self._reloader.stop()
self._reloader.join()
class KeyringModificationListener(FileSystemEventHandler):
"""A filesystem event listener that triggers key cache reload."""
def __init__(self, keyring: FileKeyRing):
"""Initialise a listener with a callback to be executed upon each change."""
self._keyring = keyring
def handle(self, event: FileSystemEvent):
"""Reload keys upon FS event."""
LOG.debug('FS event: %s, %s', event.event_type, event.src_path)
if 'pubring.kbx' in event.src_path:
LOG.info('Reloading %s on event: %s', self._keyring, event)
self._keyring.reload()
# All methods should do the same: reload the key cache.
# on_created = handle
# on_deleted = handle
on_modified = handle
def init_reloader(keyring_dir: str, reloader) -> KeyringModificationListener:
"""Initialise a reloader for the keyring."""
listener = KeyringModificationListener(reloader)
observer = Observer()
observer.schedule(listener, keyring_dir, recursive=False)
return observer
def freeze_and_load_keys():
"""Load and return keys.
Doesn't refresh the keys when they change on disk.
'"""
keyring_dir = conf.get_item('gpg', 'keyhome')
keyring = FileKeyRing(keyring_dir)
return run(keyring.freeze_identities())

179
lacre/keyring.py
View file

@ -4,172 +4,25 @@ IMPORTANT: This module has to be loaded _after_ initialisation of the logging
module.
"""
import lacre.text as text
import lacre.config as conf
from lacre._keyringcommon import KeyRing, KeyCache
import lacre.filekeyring as fk
import lacre.dbkeyring as dbk
import logging
from os import stat
from watchdog.events import FileSystemEventHandler, FileSystemEvent
from asyncio import Semaphore, create_task, get_event_loop, run
import copy
import GnuPG
LOG = logging.getLogger(__name__)
def _sanitize(keys):
sanitize = text.choose_sanitizer(conf.get_item('default', 'mail_case_insensitive'))
return {fingerprint: sanitize(keys[fingerprint]) for fingerprint in keys}
class KeyCacheMisconfiguration(Exception):
"""Exception used to signal that KeyCache is misconfigured."""
class KeyCache:
"""A store for OpenPGP keys.
Key case is sanitised while loading from GnuPG if so
configured. See mail_case_insensitive parameter in section
[default].
"""
def __init__(self, keys: dict = None):
"""Initialise an empty cache.
With keyring_dir given, set location of the directory from which keys should be loaded.
"""
self._keys = keys
def __getitem__(self, fingerpring):
"""Look up email assigned to the given fingerprint."""
return self._keys[fingerpring]
def __setitem__(self, fingerprint, email):
"""Assign an email to a fingerpring, overwriting it if it was already present."""
self._keys[fingerprint] = email
def __contains__(self, fingerprint):
"""Check if the given fingerprint is assigned to an email."""
# This method has to be present for KeyCache to be a dict substitute.
# See mailgate, function _identify_gpg_recipients.
return fingerprint in self._keys
def has_email(self, email):
"""Check if cache contains a key assigned to the given email."""
return email in self._keys.values()
def __repr__(self):
"""Return text representation of this object."""
details = ' '.join(self._keys.keys())
return '<KeyCache %s>' % (details)
class KeyRing:
"""A high-level adapter for GnuPG-maintained keyring directory.
Its role is to keep a cache of keys present in the keyring,
reload it when necessary and produce static copies of
fingerprint=>email maps.
"""
def __init__(self, path: str, loop=None):
"""Initialise the adapter."""
self._path = path
self._keys = self._load_and_sanitize()
self._sema = Semaphore()
self._last_mod = None
self._loop = loop or get_event_loop()
def _load_and_sanitize(self):
keys = self._load_keyring_from(self._path)
return _sanitize(keys)
def _load_keyring_from(self, keyring_dir):
return GnuPG.public_keys(keyring_dir)
async def freeze_identities(self) -> KeyCache:
"""Return a static, async-safe copy of the identity map."""
async with self._sema:
keys = copy.deepcopy(self._keys)
return KeyCache(keys)
def load(self):
"""Load keyring, replacing any previous contents of the cache."""
LOG.debug('Reloading keys...')
tsk = create_task(self._load(), 'LoadTask')
self._loop.run_until_complete(tsk)
async def _load(self):
last_mod = self._read_mod_time()
LOG.debug(f'Keyring was last modified: {last_mod}')
if self._is_modified(last_mod):
LOG.debug('Keyring has been modified')
async with self._sema:
LOG.debug('About to re-load the keyring')
self.replace_keyring(self._load_keyring_from(self._path))
else:
LOG.debug('Keyring not modified recently, continuing')
self._last_mod = self._read_mod_time()
reload = load
def replace_keyring(self, keys: dict):
"""Overwrite previously stored key cache with KEYS."""
keys = _sanitize(keys)
LOG.info(f'Storing {len(keys)} keys')
self._keys = keys
def _read_mod_time(self) -> int:
# (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)
# 0 1 2 3 4 5 6 7 8 9
MTIME = 8
st = stat(self._path)
return st[MTIME]
def _is_modified(self, last_mod):
if self._last_mod is None:
LOG.debug('Keyring not loaded before')
return True
elif self._last_mod != last_mod:
LOG.debug('Keyring directory mtime changed')
return True
else:
LOG.debug('Keyring not modified ')
return False
def __repr__(self) -> str:
"""Return text representation of this keyring."""
return '<KeyRing path=%s last_mod=%d>' % (self._path, self._last_mod)
class KeyringModificationListener(FileSystemEventHandler):
"""A filesystem event listener that triggers key cache reload."""
def __init__(self, keyring: KeyRing):
"""Initialise a listener with a callback to be executed upon each change."""
self._keyring = keyring
def handle(self, event: FileSystemEvent):
"""Reload keys upon FS event."""
LOG.debug('FS event: %s, %s', event.event_type, event.src_path)
if 'pubring.kbx' in event.src_path:
LOG.info('Reloading %s on event: %s', self._keyring, event)
self._keyring.reload()
# All methods should do the same: reload the key cache.
# on_created = handle
# on_deleted = handle
on_modified = handle
def freeze_and_load_keys():
"""Load and return keys.
Doesn't refresh the keys when they change on disk.
'"""
keyring_dir = conf.get_item('gpg', 'keyhome')
keyring = KeyRing(keyring_dir)
return run(keyring.freeze_identities())
def init_keyring(mode, **kwargs) -> KeyRing:
"""Initialise appropriate type of keyring."""
if mode == 'file' and 'loop' in kwargs:
path = conf.get_item('gpg', 'keyhome')
LOG.info('Initialising pubring.kbx-based keyring from %s with cache', path)
return fk.FileKeyRing(path, kwargs['loop'])
elif mode == 'database':
url = conf.get_item('keyring', 'url')
schema = dbk.KeyRingSchema()
LOG.info('Initialising database keyring from %s', url)
return dbk.DatabaseKeyRing(url, schema)
else:
LOG.error('Unsupported type of keyring: %s', mode)

5
test/gpg-mailgate-daemon-test.conf
View file

@ -8,6 +8,11 @@ date_format = ISO
keyhome = test/keyhome
cache_refresh_minutes = 1
[keyring]
#type = database
url = sqlite:///test/lacre.db
type = file
[smime]
cert_path = test/certs

16
test/modules/test_dbkeyring.py Normal file
View file

@ -0,0 +1,16 @@
"""Tests for lacre.dbkeyring."""
import lacre.dbkeyring as dbk
import unittest
class LacreDbKeyringTest(unittest.TestCase):
def test_load_keys(self):
db_url = 'sqlite:///test/lacre.db'
schema = dbk.KeyRingSchema()
db = dbk.DatabaseKeyRing(db_url, schema)
all = db.load()
self.assertTrue('1CD245308F0963D038E88357973CF4D9387C44D7' in all)
self.assertTrue(all.has_email('alice@disposlab'))

42
test/utils/schema.py
View file

@ -3,23 +3,27 @@ import sqlalchemy
from sqlalchemy.sql import insert
def define_db_schema():
meta = sqlalchemy.MetaData()
meta = sqlalchemy.MetaData()
gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta,
sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
sqlalchemy.Column('email', sqlalchemy.String(256)),
sqlalchemy.Column('publickey', sqlalchemy.Text),
sqlalchemy.Column('confirm', sqlalchemy.String(32)),
sqlalchemy.Column('status', sqlalchemy.Integer),
sqlalchemy.Column('time', sqlalchemy.DateTime))
gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta,
sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True),
sqlalchemy.Column('email', sqlalchemy.String(256)),
sqlalchemy.Column('publickey', sqlalchemy.Text),
sqlalchemy.Column('confirm', sqlalchemy.String(32)),
sqlalchemy.Column('status', sqlalchemy.Integer),
sqlalchemy.Column('time', sqlalchemy.DateTime))
return (meta, gpgmw_keys)
identities = sqlalchemy.Table('identities', meta,
sqlalchemy.Column('email', sqlalchemy.String(256), index=True),
sqlalchemy.Column('key_id', sqlalchemy.String(64), index=True))
return (meta, gpgmw_keys, identities)
if len(sys.argv) != 2:
print("ERROR: output database missing")
sys.exit(1)
print("ERROR: output database missing")
sys.exit(1)
(meta, gpgmw_keys) = define_db_schema()
(meta, gpgmw_keys, identities) = define_db_schema()
dbname = sys.argv[1]
test_db = sqlalchemy.create_engine(f"sqlite:///{dbname}")
@ -31,7 +35,7 @@ conn = test_db.connect()
# Populate the database with dummy data
conn.execute(gpgmw_keys.insert(), [
{"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
{"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
\n\
mQGNBGDYY5oBDAC+HAVjA05jsIpHfQ2KQ9m2olo1Qnlk+dkjD+Gagxj1ACezyiGL\n\
cfZfoE/MJYLCH9yPcX1fUIAPwdAyfJKlvkVcz+MhEpgl3aP3NM2L2unSx3v9ZFwT\n\
@ -73,7 +77,7 @@ pw==\n\
=Tbwz\n\
-----END PGP PUBLIC KEY BLOCK-----\
", "status": 0, "confirm": "", "time": None},
{"id": 2, "email": "bob@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
{"id": 2, "email": "bob@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
\n\
mDMEYdTFkRYJKwYBBAHaRw8BAQdA2tgdP1pMt3cv3XAW7ov5AFn74mMZvyTksp9Q\n\
eO1PkpK0GkJvYiBGb29iYXIgPGJvYkBkaXNwb3NsYWI+iJYEExYIAD4WIQQZz0tH\n\
@ -87,5 +91,11 @@ OjjB6xRD0Q2FN+alsNGCtdutAs18AZ5l33RMzws=\n\
=wWoq\n\
-----END PGP PUBLIC KEY BLOCK-----\
", "status": 0, "confirm": "", "time": None},
{"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 0, "confirm": "", "time": None}
])
{"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 0, "confirm": "", "time": None}
])
conn.execute(identities.insert(), [
{'key_id': '1CD245308F0963D038E88357973CF4D9387C44D7', 'email': 'alice@disposlab'},
{'key_id': '19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67', 'email': 'bob@disposlab'},
{'key_id': '530B1BB2D0CC7971648198BBA4774E507D3AF5BC', 'email': 'evan@disposlab'}
])