From 41442e5b590b1eb583795e5a4dfbe1e1044a9a29 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sat, 30 Sep 2023 22:38:33 +0200 Subject: [PATCH] Add basic support for RDBMS-based keyring --- gpg-mailgate.conf.sample | 13 +++ lacre/_keyringcommon.py | 57 +++++++++ lacre/daemon.py | 21 +--- lacre/dbkeyring.py | 47 ++++++++ lacre/filekeyring.py | 149 ++++++++++++++++++++++++ lacre/keyring.py | 179 +++-------------------------- test/gpg-mailgate-daemon-test.conf | 5 + test/modules/test_dbkeyring.py | 16 +++ test/utils/schema.py | 42 ++++--- 9 files changed, 335 insertions(+), 194 deletions(-) create mode 100644 lacre/_keyringcommon.py create mode 100644 lacre/dbkeyring.py create mode 100644 lacre/filekeyring.py create mode 100644 test/modules/test_dbkeyring.py diff --git a/gpg-mailgate.conf.sample b/gpg-mailgate.conf.sample index 01a1d9c..75c90bd 100644 --- a/gpg-mailgate.conf.sample +++ b/gpg-mailgate.conf.sample @@ -27,6 +27,19 @@ mail_case_insensitive = no # i.e. have mode 700. keyhome = /var/gpgmailgate/.gnupg +[keyring] +# Two options available: +# +# - file -- use GnuPG's pubring.kbx file as the only key store and cache its +# contents, reloading it on file modifications. +# +# - database -- use a relational database to store identities (when used, you +# must specify url parameter to specify which database to connect to). +type = file + +# Only required when type==DatabaseKeyRing, specifies the database URL. +#url = file:///path/to/sqlite.db + [smime] # the directory for the S/MIME certificate files cert_path = /var/gpgmailgate/smime diff --git a/lacre/_keyringcommon.py b/lacre/_keyringcommon.py new file mode 100644 index 0000000..5467dc6 --- /dev/null +++ b/lacre/_keyringcommon.py @@ -0,0 +1,57 @@ +class KeyCache: + """A store for OpenPGP keys. + + Key case is sanitised while loading from GnuPG if so + configured. See mail_case_insensitive parameter in section + [default]. + """ + + def __init__(self, keys: dict = None): + """Initialise an empty cache. + + With keyring_dir given, set location of the directory from which keys should be loaded. + """ + self._keys = keys + + def __getitem__(self, fingerpring): + """Look up email assigned to the given fingerprint.""" + return self._keys[fingerpring] + + def __setitem__(self, fingerprint, email): + """Assign an email to a fingerpring, overwriting it if it was already present.""" + self._keys[fingerprint] = email + + def __contains__(self, fingerprint): + """Check if the given fingerprint is assigned to an email.""" + # This method has to be present for KeyCache to be a dict substitute. + # See mailgate, function _identify_gpg_recipients. + return fingerprint in self._keys + + def has_email(self, email): + """Check if cache contains a key assigned to the given email.""" + return email in self._keys.values() + + def __repr__(self): + """Return text representation of this object.""" + details = ' '.join(self._keys.keys()) + return '' % (details) + + +class KeyRing: + """Contract to be implemented by a key-store (a.k.a. keyring).""" + + def load(self): + """Load keyring, replacing any previous contents of the cache.""" + raise NotImplementedError('KeyRing.load not implemented') + + async def freeze_identities(self) -> KeyCache: + """Return a static, async-safe copy of the identity map.""" + raise NotImplementedError('KeyRing.load not implemented') + + def post_init_hook(self): + """Lets the keyring perform additional operations following its initialisation.""" + pass + + def shutdown(self): + """Lets the keyring perform operations prior to shutting down.""" + pass diff --git a/lacre/daemon.py b/lacre/daemon.py index 7adfa6c..b0d2bb8 100644 --- a/lacre/daemon.py +++ b/lacre/daemon.py @@ -11,7 +11,6 @@ import asyncio import email from email.policy import SMTPUTF8 import time -from watchdog.observers import Observer # Load configuration and init logging, in this order. Only then can we load # the last Lacre module, i.e. lacre.mailgate. @@ -106,13 +105,6 @@ def _init_controller(keys: kcache.KeyRing, max_body_bytes=None, tout: float = 5) data_size_limit=max_body_bytes) -def _init_reloader(keyring_dir: str, reloader) -> kcache.KeyringModificationListener: - listener = kcache.KeyringModificationListener(reloader) - observer = Observer() - observer.schedule(listener, keyring_dir, recursive=False) - return observer - - def _validate_config(): missing = conf.validate_config() if missing: @@ -138,12 +130,12 @@ def _main(): loop = asyncio.get_event_loop() - keyring = kcache.KeyRing(keyring_path, loop) - controller = _init_controller(keyring, max_data_bytes) - reloader = _init_reloader(keyring_path, keyring) + mode = conf.get_item('keyring', 'type') - LOG.info(f'Watching keyring directory {keyring_path}...') - reloader.start() + keyring = kcache.init_keyring(mode, loop = loop) + controller = _init_controller(keyring, max_data_bytes) + + keyring.post_init_hook() LOG.info('Starting the daemon...') controller.start() @@ -156,8 +148,7 @@ def _main(): LOG.exception('Unexpected exception caught, your system may be unstable') finally: LOG.info('Shutting down keyring watcher and the daemon...') - reloader.stop() - reloader.join() + keyring.shutdown() controller.stop() LOG.info("Done") diff --git a/lacre/dbkeyring.py b/lacre/dbkeyring.py new file mode 100644 index 0000000..e6328ea --- /dev/null +++ b/lacre/dbkeyring.py @@ -0,0 +1,47 @@ +"""Database-backed keyring implementation.""" + +from lacre._keyringcommon import KeyRing, KeyCache +import logging +import sqlalchemy +from sqlalchemy.sql import select + +LOG = logging.getLogger(__name__) + + +class KeyRingSchema: + def __init__(self): + self._meta = sqlalchemy.MetaData() + self._id_table = self._identities() + + def _identities(self): + lacre_id = sqlalchemy.Table('identities', self._meta, + sqlalchemy.Column('email', sqlalchemy.String(256), index=True), + sqlalchemy.Column('key_id', sqlalchemy.String(64), index=True)) + return lacre_id + + def identities(self): + return self._id_table + + +class DatabaseKeyRing(KeyRing): + """Database-backed key storage.""" + + def __init__(self, database_url, schema: KeyRingSchema): + self._schema = schema + self._engine = sqlalchemy.create_engine(database_url) + self._connection = self._engine.connect() + + def load(self): + """Do nothing, database contents doesn't need to be cached.""" + pass + + async def freeze_identities(self) -> KeyCache: + """Return a static, async-safe copy of the identity map.""" + return self._load_identities() + + def _load_identities(self) -> KeyCache: + identities = self._schema.identities() + all_identities = select(identities.c.key_id, identities.c.email) + result = self._connection.execute(all_identities) + LOG.debug('Retrieving all keys') + return KeyCache({key_id: email for key_id, email in result}) diff --git a/lacre/filekeyring.py b/lacre/filekeyring.py new file mode 100644 index 0000000..65f3173 --- /dev/null +++ b/lacre/filekeyring.py @@ -0,0 +1,149 @@ +"""File-based keyring. + +It's a wrapper over GnuPG module that just executes gpg command. +'""" + +import logging +import GnuPG +import copy +from os import stat +from watchdog.events import FileSystemEventHandler, FileSystemEvent +from watchdog.observers import Observer +from asyncio import Semaphore, create_task, get_event_loop, run +import lacre.text as text +import lacre.config as conf +from lacre._keyringcommon import KeyRing, KeyCache + +LOG = logging.getLogger(__name__) + + +def _sanitize(keys): + sanitize = text.choose_sanitizer(conf.get_item('default', 'mail_case_insensitive')) + return {fingerprint: sanitize(keys[fingerprint]) for fingerprint in keys} + + +class FileKeyRing(KeyRing): + """A high-level adapter for GnuPG-maintained keyring directory. + + Its role is to keep a cache of keys present in the keyring, + reload it when necessary and produce static copies of + fingerprint=>email maps. + """ + + def __init__(self, path: str, loop=None): + """Initialise the adapter.""" + self._path = path + self._keys = self._load_and_sanitize() + self._sema = Semaphore() + self._last_mod = None + self._loop = loop or get_event_loop() + + def _load_and_sanitize(self): + keys = self._load_keyring_from(self._path) + return _sanitize(keys) + + def _load_keyring_from(self, keyring_dir): + return GnuPG.public_keys(keyring_dir) + + async def freeze_identities(self) -> KeyCache: + """Return a static, async-safe copy of the identity map.""" + async with self._sema: + keys = copy.deepcopy(self._keys) + return KeyCache(keys) + + def load(self): + """Load keyring, replacing any previous contents of the cache.""" + LOG.debug('Reloading keys...') + tsk = create_task(self._load(), 'LoadTask') + self._loop.run_until_complete(tsk) + + async def _load(self): + last_mod = self._read_mod_time() + LOG.debug(f'Keyring was last modified: {last_mod}') + if self._is_modified(last_mod): + LOG.debug('Keyring has been modified') + async with self._sema: + LOG.debug('About to re-load the keyring') + self.replace_keyring(self._load_keyring_from(self._path)) + else: + LOG.debug('Keyring not modified recently, continuing') + + self._last_mod = self._read_mod_time() + + reload = load + + def replace_keyring(self, keys: dict): + """Overwrite previously stored key cache with KEYS.""" + keys = _sanitize(keys) + + LOG.info(f'Storing {len(keys)} keys') + self._keys = keys + + def _read_mod_time(self) -> int: + # (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) + # 0 1 2 3 4 5 6 7 8 9 + MTIME = 8 + st = stat(self._path) + return st[MTIME] + + def _is_modified(self, last_mod): + if self._last_mod is None: + LOG.debug('Keyring not loaded before') + return True + elif self._last_mod != last_mod: + LOG.debug('Keyring directory mtime changed') + return True + else: + LOG.debug('Keyring not modified ') + return False + + def __repr__(self) -> str: + """Return text representation of this keyring.""" + return '' % (self._path, self._last_mod) + + def post_init_hook(self): + self._reloader = init_reloader(self._path, self) + LOG.info(f'Watching keyring directory {self._path}...') + self._reloader.start() + + def shutdown(self): + self._reloader.stop() + self._reloader.join() + + +class KeyringModificationListener(FileSystemEventHandler): + """A filesystem event listener that triggers key cache reload.""" + + def __init__(self, keyring: FileKeyRing): + """Initialise a listener with a callback to be executed upon each change.""" + self._keyring = keyring + + def handle(self, event: FileSystemEvent): + """Reload keys upon FS event.""" + LOG.debug('FS event: %s, %s', event.event_type, event.src_path) + if 'pubring.kbx' in event.src_path: + LOG.info('Reloading %s on event: %s', self._keyring, event) + self._keyring.reload() + + # All methods should do the same: reload the key cache. + # on_created = handle + # on_deleted = handle + on_modified = handle + + +def init_reloader(keyring_dir: str, reloader) -> KeyringModificationListener: + """Initialise a reloader for the keyring.""" + listener = KeyringModificationListener(reloader) + observer = Observer() + observer.schedule(listener, keyring_dir, recursive=False) + return observer + + +def freeze_and_load_keys(): + """Load and return keys. + + Doesn't refresh the keys when they change on disk. + '""" + keyring_dir = conf.get_item('gpg', 'keyhome') + keyring = FileKeyRing(keyring_dir) + return run(keyring.freeze_identities()) diff --git a/lacre/keyring.py b/lacre/keyring.py index 6919777..a43a8d6 100644 --- a/lacre/keyring.py +++ b/lacre/keyring.py @@ -4,172 +4,25 @@ IMPORTANT: This module has to be loaded _after_ initialisation of the logging module. """ -import lacre.text as text import lacre.config as conf +from lacre._keyringcommon import KeyRing, KeyCache +import lacre.filekeyring as fk +import lacre.dbkeyring as dbk import logging -from os import stat -from watchdog.events import FileSystemEventHandler, FileSystemEvent -from asyncio import Semaphore, create_task, get_event_loop, run -import copy - -import GnuPG LOG = logging.getLogger(__name__) -def _sanitize(keys): - sanitize = text.choose_sanitizer(conf.get_item('default', 'mail_case_insensitive')) - return {fingerprint: sanitize(keys[fingerprint]) for fingerprint in keys} - - -class KeyCacheMisconfiguration(Exception): - """Exception used to signal that KeyCache is misconfigured.""" - - -class KeyCache: - """A store for OpenPGP keys. - - Key case is sanitised while loading from GnuPG if so - configured. See mail_case_insensitive parameter in section - [default]. - """ - - def __init__(self, keys: dict = None): - """Initialise an empty cache. - - With keyring_dir given, set location of the directory from which keys should be loaded. - """ - self._keys = keys - - def __getitem__(self, fingerpring): - """Look up email assigned to the given fingerprint.""" - return self._keys[fingerpring] - - def __setitem__(self, fingerprint, email): - """Assign an email to a fingerpring, overwriting it if it was already present.""" - self._keys[fingerprint] = email - - def __contains__(self, fingerprint): - """Check if the given fingerprint is assigned to an email.""" - # This method has to be present for KeyCache to be a dict substitute. - # See mailgate, function _identify_gpg_recipients. - return fingerprint in self._keys - - def has_email(self, email): - """Check if cache contains a key assigned to the given email.""" - return email in self._keys.values() - - def __repr__(self): - """Return text representation of this object.""" - details = ' '.join(self._keys.keys()) - return '' % (details) - - -class KeyRing: - """A high-level adapter for GnuPG-maintained keyring directory. - - Its role is to keep a cache of keys present in the keyring, - reload it when necessary and produce static copies of - fingerprint=>email maps. - """ - - def __init__(self, path: str, loop=None): - """Initialise the adapter.""" - self._path = path - self._keys = self._load_and_sanitize() - self._sema = Semaphore() - self._last_mod = None - self._loop = loop or get_event_loop() - - def _load_and_sanitize(self): - keys = self._load_keyring_from(self._path) - return _sanitize(keys) - - def _load_keyring_from(self, keyring_dir): - return GnuPG.public_keys(keyring_dir) - - async def freeze_identities(self) -> KeyCache: - """Return a static, async-safe copy of the identity map.""" - async with self._sema: - keys = copy.deepcopy(self._keys) - return KeyCache(keys) - - def load(self): - """Load keyring, replacing any previous contents of the cache.""" - LOG.debug('Reloading keys...') - tsk = create_task(self._load(), 'LoadTask') - self._loop.run_until_complete(tsk) - - async def _load(self): - last_mod = self._read_mod_time() - LOG.debug(f'Keyring was last modified: {last_mod}') - if self._is_modified(last_mod): - LOG.debug('Keyring has been modified') - async with self._sema: - LOG.debug('About to re-load the keyring') - self.replace_keyring(self._load_keyring_from(self._path)) - else: - LOG.debug('Keyring not modified recently, continuing') - - self._last_mod = self._read_mod_time() - - reload = load - - def replace_keyring(self, keys: dict): - """Overwrite previously stored key cache with KEYS.""" - keys = _sanitize(keys) - - LOG.info(f'Storing {len(keys)} keys') - self._keys = keys - - def _read_mod_time(self) -> int: - # (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) - # 0 1 2 3 4 5 6 7 8 9 - MTIME = 8 - st = stat(self._path) - return st[MTIME] - - def _is_modified(self, last_mod): - if self._last_mod is None: - LOG.debug('Keyring not loaded before') - return True - elif self._last_mod != last_mod: - LOG.debug('Keyring directory mtime changed') - return True - else: - LOG.debug('Keyring not modified ') - return False - - def __repr__(self) -> str: - """Return text representation of this keyring.""" - return '' % (self._path, self._last_mod) - - -class KeyringModificationListener(FileSystemEventHandler): - """A filesystem event listener that triggers key cache reload.""" - - def __init__(self, keyring: KeyRing): - """Initialise a listener with a callback to be executed upon each change.""" - self._keyring = keyring - - def handle(self, event: FileSystemEvent): - """Reload keys upon FS event.""" - LOG.debug('FS event: %s, %s', event.event_type, event.src_path) - if 'pubring.kbx' in event.src_path: - LOG.info('Reloading %s on event: %s', self._keyring, event) - self._keyring.reload() - - # All methods should do the same: reload the key cache. - # on_created = handle - # on_deleted = handle - on_modified = handle - - -def freeze_and_load_keys(): - """Load and return keys. - - Doesn't refresh the keys when they change on disk. - '""" - keyring_dir = conf.get_item('gpg', 'keyhome') - keyring = KeyRing(keyring_dir) - return run(keyring.freeze_identities()) +def init_keyring(mode, **kwargs) -> KeyRing: + """Initialise appropriate type of keyring.""" + if mode == 'file' and 'loop' in kwargs: + path = conf.get_item('gpg', 'keyhome') + LOG.info('Initialising pubring.kbx-based keyring from %s with cache', path) + return fk.FileKeyRing(path, kwargs['loop']) + elif mode == 'database': + url = conf.get_item('keyring', 'url') + schema = dbk.KeyRingSchema() + LOG.info('Initialising database keyring from %s', url) + return dbk.DatabaseKeyRing(url, schema) + else: + LOG.error('Unsupported type of keyring: %s', mode) diff --git a/test/gpg-mailgate-daemon-test.conf b/test/gpg-mailgate-daemon-test.conf index 29331a0..13bd99e 100644 --- a/test/gpg-mailgate-daemon-test.conf +++ b/test/gpg-mailgate-daemon-test.conf @@ -8,6 +8,11 @@ date_format = ISO keyhome = test/keyhome cache_refresh_minutes = 1 +[keyring] +#type = database +url = sqlite:///test/lacre.db +type = file + [smime] cert_path = test/certs diff --git a/test/modules/test_dbkeyring.py b/test/modules/test_dbkeyring.py new file mode 100644 index 0000000..2cc0783 --- /dev/null +++ b/test/modules/test_dbkeyring.py @@ -0,0 +1,16 @@ +"""Tests for lacre.dbkeyring.""" + +import lacre.dbkeyring as dbk + +import unittest + +class LacreDbKeyringTest(unittest.TestCase): + def test_load_keys(self): + db_url = 'sqlite:///test/lacre.db' + schema = dbk.KeyRingSchema() + db = dbk.DatabaseKeyRing(db_url, schema) + + all = db.load() + + self.assertTrue('1CD245308F0963D038E88357973CF4D9387C44D7' in all) + self.assertTrue(all.has_email('alice@disposlab')) diff --git a/test/utils/schema.py b/test/utils/schema.py index 5e99760..641cc08 100644 --- a/test/utils/schema.py +++ b/test/utils/schema.py @@ -3,23 +3,27 @@ import sqlalchemy from sqlalchemy.sql import insert def define_db_schema(): - meta = sqlalchemy.MetaData() + meta = sqlalchemy.MetaData() - gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True), - sqlalchemy.Column('email', sqlalchemy.String(256)), - sqlalchemy.Column('publickey', sqlalchemy.Text), - sqlalchemy.Column('confirm', sqlalchemy.String(32)), - sqlalchemy.Column('status', sqlalchemy.Integer), - sqlalchemy.Column('time', sqlalchemy.DateTime)) + gpgmw_keys = sqlalchemy.Table('gpgmw_keys', meta, + sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True), + sqlalchemy.Column('email', sqlalchemy.String(256)), + sqlalchemy.Column('publickey', sqlalchemy.Text), + sqlalchemy.Column('confirm', sqlalchemy.String(32)), + sqlalchemy.Column('status', sqlalchemy.Integer), + sqlalchemy.Column('time', sqlalchemy.DateTime)) - return (meta, gpgmw_keys) + identities = sqlalchemy.Table('identities', meta, + sqlalchemy.Column('email', sqlalchemy.String(256), index=True), + sqlalchemy.Column('key_id', sqlalchemy.String(64), index=True)) + + return (meta, gpgmw_keys, identities) if len(sys.argv) != 2: - print("ERROR: output database missing") - sys.exit(1) + print("ERROR: output database missing") + sys.exit(1) -(meta, gpgmw_keys) = define_db_schema() +(meta, gpgmw_keys, identities) = define_db_schema() dbname = sys.argv[1] test_db = sqlalchemy.create_engine(f"sqlite:///{dbname}") @@ -31,7 +35,7 @@ conn = test_db.connect() # Populate the database with dummy data conn.execute(gpgmw_keys.insert(), [ - {"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\ + {"id": 1, "email": "alice@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\ \n\ mQGNBGDYY5oBDAC+HAVjA05jsIpHfQ2KQ9m2olo1Qnlk+dkjD+Gagxj1ACezyiGL\n\ cfZfoE/MJYLCH9yPcX1fUIAPwdAyfJKlvkVcz+MhEpgl3aP3NM2L2unSx3v9ZFwT\n\ @@ -73,7 +77,7 @@ pw==\n\ =Tbwz\n\ -----END PGP PUBLIC KEY BLOCK-----\ ", "status": 0, "confirm": "", "time": None}, - {"id": 2, "email": "bob@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\ + {"id": 2, "email": "bob@disposlab", "publickey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\ \n\ mDMEYdTFkRYJKwYBBAHaRw8BAQdA2tgdP1pMt3cv3XAW7ov5AFn74mMZvyTksp9Q\n\ eO1PkpK0GkJvYiBGb29iYXIgPGJvYkBkaXNwb3NsYWI+iJYEExYIAD4WIQQZz0tH\n\ @@ -87,5 +91,11 @@ OjjB6xRD0Q2FN+alsNGCtdutAs18AZ5l33RMzws=\n\ =wWoq\n\ -----END PGP PUBLIC KEY BLOCK-----\ ", "status": 0, "confirm": "", "time": None}, - {"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 0, "confirm": "", "time": None} - ]) + {"id": 3, "email": "cecil@lacre.io", "publickey": "RUBBISH", "status": 0, "confirm": "", "time": None} + ]) + +conn.execute(identities.insert(), [ + {'key_id': '1CD245308F0963D038E88357973CF4D9387C44D7', 'email': 'alice@disposlab'}, + {'key_id': '19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67', 'email': 'bob@disposlab'}, + {'key_id': '530B1BB2D0CC7971648198BBA4774E507D3AF5BC', 'email': 'evan@disposlab'} + ])