From a9506805fb90e197723375011648925117bc0821 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Thu, 4 Nov 2021 22:39:02 +0100 Subject: [PATCH 01/27] Write a test mail relay Provide a simple Python script that would linsten on a given port and print to standard output any message received via SMTP on that port. This script will then be used to automatically test gpg-mailgate with different scenarios (unknown recipient key, RSA key, elliptic curve key). --- test/relay.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 test/relay.py diff --git a/test/relay.py b/test/relay.py new file mode 100644 index 0000000..cfbe02a --- /dev/null +++ b/test/relay.py @@ -0,0 +1,68 @@ +#!/usr/local/bin/python2 +# +# This quick-and-dirty script supports only the happy case of SMTP session, +# i.e. what gpg-mailgate/gpg-lacre needs to deliver encrypted email. +# +# It listens on the port given as the only command-line argument and consumes a +# message, then prints it to standard output. The goal is to be able to +# compare that output with expected clear-text or encrypted message body. +# + +import sys +import socket + + +BUFFER_SIZE = 4096 +EOM = "\r\n.\r\n" +LAST_LINE = -3 + + +def welcome(msg): + return "220 %s\r\n" % (msg) + +def ok(msg = "OK"): + return "250 %s\r\n" % (msg) + +def provide_message(): + return "354 Enter a message, ending it with a '.' on a line by itself\r\n" + +def receive_and_confirm(session): + session.recv(BUFFER_SIZE) + session.sendall(ok()) + +def serve(port): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('', port)) + s.listen(1) + + (conn, addr) = s.accept() + conn.sendall(welcome("TEST SERVER")) + + receive_and_confirm(conn) # Ignore HELO/EHLO + receive_and_confirm(conn) # Ignore sender address + receive_and_confirm(conn) # Ignore recipient address + + data = conn.recv(BUFFER_SIZE) + conn.sendall(provide_message()) + + # Consume until we get ., the end-of-message marker. + message = '' + while not message.endswith(EOM): + message += conn.recv(BUFFER_SIZE) + conn.sendall(ok("OK, id=test")) + + # Trim EOM marker as we're only interested in the message body. + return message[:-len(EOM)] + +def error(msg): + print "ERROR: %s" % (msg) + sys.exit(1) + + +if len(sys.argv) < 2: + error("Usage: relay.py PORT_NUMBER") + +port = int(sys.argv[1]) +body = serve(port) + +print body From f3f56a47bcffaa99550db789f8d89aa9e97edc1c Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Mon, 8 Nov 2021 22:52:22 +0100 Subject: [PATCH 02/27] Implement an E2E testing script --- test/e2e_test.py | 110 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 test/e2e_test.py diff --git a/test/e2e_test.py b/test/e2e_test.py new file mode 100644 index 0000000..e37f7ff --- /dev/null +++ b/test/e2e_test.py @@ -0,0 +1,110 @@ +#!/usr/local/bin/python2 + +import os +import sys + +import difflib + +import ConfigParser +import logging + +TEST_PORT = 2500 + +EOL = "\n" + +RELAY_SCRIPT = "test/relay.py" +CONFIG_FILE = "test/gpg-mailgate.conf" + +PYTHON_BIN = "python2.7" + +def build_config(config): + cp = ConfigParser.ConfigParser() + + cp.add_section("logging") + cp.set("logging", "file", "/dev/stout") + cp.set("logging", "verbose", "yes") + + cp.add_section("gpg") + cp.set("gpg", "keyhome", config["gpg_keyhome"]) + + cp.add_section("smime") + cp.set("smime", "cert_path", config["smime_certpath"]) + + cp.add_section("relay") + cp.set("relay", "host", "localhost") + cp.set("relay", "port", config["port"]) + + logging.debug("Created config with keyhome=%s, cert_path=%s and relay at port %d" % + (config["gpg_keyhome"], config["smime_certpath"], config["port"])) + return cp + +def write_test_config(outfile, **config): + logging.debug("Generating configuration with %s" % repr(config)) + + out = open(outfile, "w+") + cp = build_config(config) + cp.write(out) + out.close() + + logging.debug("Wrote configuration to %s" % outfile) + +def load_file(name): + f = open(name, 'r') + contents = f.read() + f.close() + + return contents + +def strip_eols(strings): + return map(lambda s: s.strip("\r"), strings) + +def compare(result, expected): + result_lines = strip_eols(result.split(EOL)) + expected_lines = strip_eols(expected.split(EOL)) + + return difflib.unified_diff(expected_lines, result_lines, + fromfile='expected', + tofile='output') + +def report_result(message_file, test_output): + expected = load_file(message_file) + diff = compare(test_output, expected) + if len(list(diff)) > 0: + print "Output and the expected message don't match:" + else: + print "Message %s processed properly" % (message_file) + for diff_line in diff: + print diff_line + +def execute_e2e_test(message_file, **kwargs): + test_command = "%s gpg-mailgate.py %s < %s" % (PYTHON_BIN, kwargs["from_addr"], message_file) + result_command = "%s %s %d" % (PYTHON_BIN, RELAY_SCRIPT, kwargs["port"]) + + logging.debug("Spawning: '%s'" % (result_command)) + pipe = os.popen(result_command, 'r') + + logging.debug("Spawning: '%s'" % (test_command)) + msgin = os.popen(test_command, 'w') + msgin.write(load_file(message_file)) + msgin.close() + + testout = pipe.read() + pipe.close() + + logging.debug("Read %d characters of test output: '%s'" % (len(testout), testout)) + + report_result(message_file, testout) + + +logging.basicConfig(filename = "e2e_test.log", + datefmt = "%Y-%m-%d %H:%M:%S", + level = logging.DEBUG) + +write_test_config(os.getcwd() + "/" + CONFIG_FILE, + port = TEST_PORT, + gpg_keyhome = "test/keyhome", + smime_certpath = "test/certs") + +execute_e2e_test("test.msg", + from_addr = "alice@localhost", + port = TEST_PORT) From 7a063a91b844b54a940be3f50ded62287d9a6f17 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Tue, 9 Nov 2021 21:25:41 +0100 Subject: [PATCH 03/27] Polish E2E testing script and make it configurable --- Makefile | 6 ++++++ gpg-mailgate.py | 7 ++++++- test/e2e.ini | 10 ++++++++++ test/e2e_test.py | 32 +++++++++++++++++++++----------- test/msgin/clear2clear.msg | 5 +++++ test/msgout/clear2clear.msg | 5 +++++ 6 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 Makefile create mode 100644 test/e2e.ini create mode 100644 test/msgin/clear2clear.msg create mode 100644 test/msgout/clear2clear.msg diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1a4bc8a --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +PYTHON = python2.7 + +.PHONY: test + +test: + $(PYTHON) test/e2e_test.py diff --git a/gpg-mailgate.py b/gpg-mailgate.py index 9830262..b39c786 100755 --- a/gpg-mailgate.py +++ b/gpg-mailgate.py @@ -39,9 +39,14 @@ import traceback from M2Crypto import BIO, Rand, SMIME, X509 from email.mime.message import MIMEMessage +# Environment variable name we read to retrieve configuration path. This is to +# enable non-root users to set up and run GPG Mailgate and to make the software +# testable. +CONFIG_PATH_ENV = "GPG_MAILGATE_CONFIG" + # Read configuration from /etc/gpg-mailgate.conf _cfg = RawConfigParser() -_cfg.read('/etc/gpg-mailgate.conf') +_cfg.read(os.getenv(CONFIG_PATH_ENV, '/etc/gpg-mailgate.conf')) cfg = dict() for sect in _cfg.sections(): cfg[sect] = dict() diff --git a/test/e2e.ini b/test/e2e.ini new file mode 100644 index 0000000..6f3574d --- /dev/null +++ b/test/e2e.ini @@ -0,0 +1,10 @@ +[relay] +port = 2500 + +[tests] +cases = 1 + +[case-1] +from = alice@localhost +in = test/msgin/clear2clear.msg +out = test/msgout/clear2clear.msg diff --git a/test/e2e_test.py b/test/e2e_test.py index e37f7ff..40964ad 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -8,8 +8,6 @@ import difflib import ConfigParser import logging -TEST_PORT = 2500 - EOL = "\n" RELAY_SCRIPT = "test/relay.py" @@ -66,17 +64,17 @@ def compare(result, expected): fromfile='expected', tofile='output') -def report_result(message_file, test_output): - expected = load_file(message_file) +def report_result(message_file, expected_file, test_output): + expected = load_file(expected_file) diff = compare(test_output, expected) if len(list(diff)) > 0: - print "Output and the expected message don't match:" + print "Output and the expected message (%s) don't match:" % (expected_file) else: print "Message %s processed properly" % (message_file) for diff_line in diff: print diff_line -def execute_e2e_test(message_file, **kwargs): +def execute_e2e_test(message_file, expected_file, **kwargs): test_command = "%s gpg-mailgate.py %s < %s" % (PYTHON_BIN, kwargs["from_addr"], message_file) result_command = "%s %s %d" % (PYTHON_BIN, RELAY_SCRIPT, kwargs["port"]) @@ -93,18 +91,30 @@ def execute_e2e_test(message_file, **kwargs): logging.debug("Read %d characters of test output: '%s'" % (len(testout), testout)) - report_result(message_file, testout) + report_result(message_file, expected_file, testout) +def load_config(): + cp = ConfigParser.ConfigParser() + cp.read("test/e2e.ini") + + return cp + + +config = load_config() logging.basicConfig(filename = "e2e_test.log", + format = "%(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s", datefmt = "%Y-%m-%d %H:%M:%S", level = logging.DEBUG) write_test_config(os.getcwd() + "/" + CONFIG_FILE, - port = TEST_PORT, + port = config.getint("relay", "port"), gpg_keyhome = "test/keyhome", smime_certpath = "test/certs") -execute_e2e_test("test.msg", - from_addr = "alice@localhost", - port = TEST_PORT) +for case_no in range(1, config.getint("tests", "cases")+1): + case_name = "case-%d" % (case_no) + + execute_e2e_test(config.get(case_name, "in"), config.get(case_name, "out"), + from_addr = config.get(case_name, "from"), + port = config.getint("relay", "port")) diff --git a/test/msgin/clear2clear.msg b/test/msgin/clear2clear.msg new file mode 100644 index 0000000..5fe7ff7 --- /dev/null +++ b/test/msgin/clear2clear.msg @@ -0,0 +1,5 @@ +From: Bob +To: Alice +Subject: Test + +Body of the message. diff --git a/test/msgout/clear2clear.msg b/test/msgout/clear2clear.msg new file mode 100644 index 0000000..5fe7ff7 --- /dev/null +++ b/test/msgout/clear2clear.msg @@ -0,0 +1,5 @@ +From: Bob +To: Alice +Subject: Test + +Body of the message. From 27d7481078a4dff419efa960734009f4817d0c7d Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Thu, 11 Nov 2021 10:57:00 +0100 Subject: [PATCH 04/27] Set up ground for E2E tests - Use an environment variable to point at the configuration file while strating gpg-mailgate.py. - Unify paths: store temporary config, logs and anything else under 'test' directory. - Configure more tests (RSA, Ed25519). - Add test descriptions to be shown before they're started. --- Makefile | 16 ++++++++++++++-- test/e2e.ini | 17 +++++++++++++++-- test/e2e_test.py | 27 +++++++++++++++++++-------- test/msgin/clear2ed.msg | 5 +++++ test/msgin/clear2rsa.msg | 5 +++++ test/msgout/clear2ed.msg | 5 +++++ test/msgout/clear2rsa.msg | 5 +++++ test/relay.py | 10 ++++++++-- 8 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 test/msgin/clear2ed.msg create mode 100644 test/msgin/clear2rsa.msg create mode 100644 test/msgout/clear2ed.msg create mode 100644 test/msgout/clear2rsa.msg diff --git a/Makefile b/Makefile index 1a4bc8a..5c50c1d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,18 @@ PYTHON = python2.7 -.PHONY: test +.PHONY: test pre-clean clean -test: +test: test/tmp test/logs pre-clean $(PYTHON) test/e2e_test.py + +pre-clean: + rm -fv test/gpg-mailgate.conf + +test/tmp: + mkdir test/tmp + +test/logs: + mkdir test/logs + +clean: pre-clean + rm -rfv test/tmp test/logs diff --git a/test/e2e.ini b/test/e2e.ini index 6f3574d..3e0899f 100644 --- a/test/e2e.ini +++ b/test/e2e.ini @@ -2,9 +2,22 @@ port = 2500 [tests] -cases = 1 +cases = 3 [case-1] -from = alice@localhost +descr = Clear text message to a user without a key +to = carlos@disposlab in = test/msgin/clear2clear.msg out = test/msgout/clear2clear.msg + +[case-2] +descr = Clear text message to a user with an RSA key +to = alice@disposlab +in = test/msgin/clear2rsa.msg +out = test/msgout/clear2rsa.msg + +[case-3] +descr = Clear text message to a user with an Ed25519 key +to = bob@disposlab +in = test/msgin/clear2ed.msg +out = test/msgout/clear2ed.msg diff --git a/test/e2e_test.py b/test/e2e_test.py index 40964ad..37ccbb0 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -8,8 +8,12 @@ import difflib import ConfigParser import logging +from time import sleep + EOL = "\n" +DELAY = 3 + RELAY_SCRIPT = "test/relay.py" CONFIG_FILE = "test/gpg-mailgate.conf" @@ -19,7 +23,7 @@ def build_config(config): cp = ConfigParser.ConfigParser() cp.add_section("logging") - cp.set("logging", "file", "/dev/stout") + cp.set("logging", "file", config["log_file"]) cp.set("logging", "verbose", "yes") cp.add_section("gpg") @@ -75,7 +79,7 @@ def report_result(message_file, expected_file, test_output): print diff_line def execute_e2e_test(message_file, expected_file, **kwargs): - test_command = "%s gpg-mailgate.py %s < %s" % (PYTHON_BIN, kwargs["from_addr"], message_file) + test_command = "GPG_MAILGATE_CONFIG=%s %s gpg-mailgate.py %s < %s" % (kwargs["config_path"], PYTHON_BIN, kwargs["to_addr"], message_file) result_command = "%s %s %d" % (PYTHON_BIN, RELAY_SCRIPT, kwargs["port"]) logging.debug("Spawning: '%s'" % (result_command)) @@ -102,19 +106,26 @@ def load_config(): config = load_config() -logging.basicConfig(filename = "e2e_test.log", - format = "%(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s", +logging.basicConfig(filename = "test/logs/e2e.log", + format = "%(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s", datefmt = "%Y-%m-%d %H:%M:%S", level = logging.DEBUG) -write_test_config(os.getcwd() + "/" + CONFIG_FILE, +config_path = os.getcwd() + "/" + CONFIG_FILE + +write_test_config(config_path, port = config.getint("relay", "port"), gpg_keyhome = "test/keyhome", - smime_certpath = "test/certs") + smime_certpath = "test/certs", + log_file = "test/logs/gpg-mailgate.log") for case_no in range(1, config.getint("tests", "cases")+1): case_name = "case-%d" % (case_no) + print "Executing: %s" % (config.get(case_name, "descr")) execute_e2e_test(config.get(case_name, "in"), config.get(case_name, "out"), - from_addr = config.get(case_name, "from"), - port = config.getint("relay", "port")) + config_path = config_path, + to_addr = config.get(case_name, "to"), + port = config.getint("relay", "port")) + + sleep(DELAY) diff --git a/test/msgin/clear2ed.msg b/test/msgin/clear2ed.msg new file mode 100644 index 0000000..5fe7ff7 --- /dev/null +++ b/test/msgin/clear2ed.msg @@ -0,0 +1,5 @@ +From: Bob +To: Alice +Subject: Test + +Body of the message. diff --git a/test/msgin/clear2rsa.msg b/test/msgin/clear2rsa.msg new file mode 100644 index 0000000..5fe7ff7 --- /dev/null +++ b/test/msgin/clear2rsa.msg @@ -0,0 +1,5 @@ +From: Bob +To: Alice +Subject: Test + +Body of the message. diff --git a/test/msgout/clear2ed.msg b/test/msgout/clear2ed.msg new file mode 100644 index 0000000..5fe7ff7 --- /dev/null +++ b/test/msgout/clear2ed.msg @@ -0,0 +1,5 @@ +From: Bob +To: Alice +Subject: Test + +Body of the message. diff --git a/test/msgout/clear2rsa.msg b/test/msgout/clear2rsa.msg new file mode 100644 index 0000000..5fe7ff7 --- /dev/null +++ b/test/msgout/clear2rsa.msg @@ -0,0 +1,5 @@ +From: Bob +To: Alice +Subject: Test + +Body of the message. diff --git a/test/relay.py b/test/relay.py index cfbe02a..eca73ec 100644 --- a/test/relay.py +++ b/test/relay.py @@ -12,6 +12,8 @@ import sys import socket +EXIT_UNAVAILABLE = 1 + BUFFER_SIZE = 4096 EOM = "\r\n.\r\n" LAST_LINE = -3 @@ -32,8 +34,12 @@ def receive_and_confirm(session): def serve(port): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind(('', port)) - s.listen(1) + try: + s.bind(('', port)) + s.listen(1) + except socket.error, e: + print "Cannot connect", e + sys.exit(EXIT_UNAVAILABLE) (conn, addr) = s.accept() conn.sendall(welcome("TEST SERVER")) From f41adc0d53d04644c6b2fa1db9385379d7ba15a1 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Thu, 11 Nov 2021 11:03:53 +0100 Subject: [PATCH 05/27] Configure git to ignore temporary files and Emacs files --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 7808c4b..6b8344e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,16 @@ pip-log.txt .tox nosetests.xml +# GPG-Mailgate test files +test/logs +test/gpg-mailgate.conf + +# Emacs files +*~ +TAGS +TAGS-Python +TAGS-PHP + # Translations *.mo From f1a799d8647f959c51f0444c5b8b8c6250b58603 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Thu, 6 Jan 2022 16:23:10 +0100 Subject: [PATCH 06/27] Adjust E2E tests to work with all scenarios Since it's not so easy to encrypt a message exactly the same way twice, we only verify if the message has been encrypted or not. Introduce minor changes to the library itself, because it doesn't work very well with modern GnuPG. Also, include GnuPG directory (pointed at by --homedir option). --- GnuPG/__init__.py | 20 ++++++++++++--- Makefile | 1 + test/e2e.ini | 6 ++--- test/e2e_test.py | 50 +++++++++++++++++++++++------------- test/keyhome/crls.d/DIR.txt | 1 + test/keyhome/pubring.kbx | Bin 0 -> 3088 bytes test/keyhome/random_seed | Bin 0 -> 600 bytes test/keyhome/tofu.db | Bin 0 -> 49152 bytes test/keyhome/trustdb.gpg | Bin 0 -> 1200 bytes test/relay.py | 8 ++++++ 10 files changed, 62 insertions(+), 24 deletions(-) create mode 100644 test/keyhome/crls.d/DIR.txt create mode 100644 test/keyhome/pubring.kbx create mode 100644 test/keyhome/random_seed create mode 100644 test/keyhome/tofu.db create mode 100644 test/keyhome/trustdb.gpg diff --git a/GnuPG/__init__.py b/GnuPG/__init__.py index c9bbee0..294f8c9 100644 --- a/GnuPG/__init__.py +++ b/GnuPG/__init__.py @@ -23,6 +23,13 @@ import subprocess import shutil import random import string +import sys + + +LINE_FINGERPRINT = 'fpr' +LINE_USER_ID = 'uid' + +POS_FINGERPRINT = 9 def private_keys( keyhome ): cmd = ['/usr/bin/gpg', '--homedir', keyhome, '--list-secret-keys', '--with-colons'] @@ -42,14 +49,21 @@ def public_keys( keyhome ): cmd = ['/usr/bin/gpg', '--homedir', keyhome, '--list-keys', '--with-colons'] p = subprocess.Popen( cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.wait() + keys = dict() + fingerprint = None + email = None for line in p.stdout.readlines(): - if line[0:3] == 'uid' or line[0:3] == 'pub': + if line[0:3] == LINE_FINGERPRINT: + fingerprint = line.split(':')[POS_FINGERPRINT] + if line[0:3] == LINE_USER_ID: if ('<' not in line or '>' not in line): continue email = line.split('<')[1].split('>')[0] - fingerprint = line.split(':')[4] + if not (fingerprint is None or email is None): keys[fingerprint] = email + fingerprint = None + email = None return keys # confirms a key has a given email address @@ -147,4 +161,4 @@ class GPGDecryptor: def _command(self): cmd = ["/usr/bin/gpg", "--trust-model", "always", "--homedir", self._keyhome, "--batch", "--yes", "--no-secmem-warning", "-a", "-d"] - return cmd \ No newline at end of file + return cmd diff --git a/Makefile b/Makefile index 5c50c1d..629a04b 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ test: test/tmp test/logs pre-clean pre-clean: rm -fv test/gpg-mailgate.conf + rm -f test/logs/*.log test/tmp: mkdir test/tmp diff --git a/test/e2e.ini b/test/e2e.ini index 3e0899f..b65d0dc 100644 --- a/test/e2e.ini +++ b/test/e2e.ini @@ -8,16 +8,16 @@ cases = 3 descr = Clear text message to a user without a key to = carlos@disposlab in = test/msgin/clear2clear.msg -out = test/msgout/clear2clear.msg +out = Body of the message. [case-2] descr = Clear text message to a user with an RSA key to = alice@disposlab in = test/msgin/clear2rsa.msg -out = test/msgout/clear2rsa.msg +out = -----BEGIN PGP MESSAGE----- [case-3] descr = Clear text message to a user with an Ed25519 key to = bob@disposlab in = test/msgin/clear2ed.msg -out = test/msgout/clear2ed.msg +out = -----BEGIN PGP MESSAGE----- diff --git a/test/e2e_test.py b/test/e2e_test.py index 37ccbb0..c730144 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -36,6 +36,10 @@ def build_config(config): cp.set("relay", "host", "localhost") cp.set("relay", "port", config["port"]) + cp.add_section("enc_keymap") + cp.set("enc_keymap", "alice@disposlab", "1CD245308F0963D038E88357973CF4D9387C44D7") + cp.set("enc_keymap", "bob@disposlab", "19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67") + logging.debug("Created config with keyhome=%s, cert_path=%s and relay at port %d" % (config["gpg_keyhome"], config["smime_certpath"], config["port"])) return cp @@ -68,18 +72,27 @@ def compare(result, expected): fromfile='expected', tofile='output') -def report_result(message_file, expected_file, test_output): - expected = load_file(expected_file) - diff = compare(test_output, expected) - if len(list(diff)) > 0: - print "Output and the expected message (%s) don't match:" % (expected_file) +def report_result(message_file, expected, test_output): + status = None + if expected in test_output: + status = "Success" else: - print "Message %s processed properly" % (message_file) - for diff_line in diff: - print diff_line + status = "Failure" -def execute_e2e_test(message_file, expected_file, **kwargs): - test_command = "GPG_MAILGATE_CONFIG=%s %s gpg-mailgate.py %s < %s" % (kwargs["config_path"], PYTHON_BIN, kwargs["to_addr"], message_file) + print "%s %s" % (message_file.ljust(30, '.'), status) + +def frozen_time_expr(timestamp): + if timestamp is None: + return "" + else: + return "GPG_FROZEN_TIME=%s" % (timestamp) + +def execute_e2e_test(message_file, expected, **kwargs): + test_command = "GPG_MAILGATE_CONFIG=%s %s gpg-mailgate.py %s < %s" % ( + kwargs["config_path"], + PYTHON_BIN, + kwargs["to_addr"], + message_file) result_command = "%s %s %d" % (PYTHON_BIN, RELAY_SCRIPT, kwargs["port"]) logging.debug("Spawning: '%s'" % (result_command)) @@ -95,16 +108,16 @@ def execute_e2e_test(message_file, expected_file, **kwargs): logging.debug("Read %d characters of test output: '%s'" % (len(testout), testout)) - report_result(message_file, expected_file, testout) + report_result(message_file, expected, testout) -def load_config(): +def load_test_config(): cp = ConfigParser.ConfigParser() cp.read("test/e2e.ini") return cp -config = load_config() +config = load_test_config() logging.basicConfig(filename = "test/logs/e2e.log", format = "%(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s", @@ -123,9 +136,10 @@ for case_no in range(1, config.getint("tests", "cases")+1): case_name = "case-%d" % (case_no) print "Executing: %s" % (config.get(case_name, "descr")) - execute_e2e_test(config.get(case_name, "in"), config.get(case_name, "out"), - config_path = config_path, - to_addr = config.get(case_name, "to"), - port = config.getint("relay", "port")) + execute_e2e_test(config.get(case_name, "in"), + config.get(case_name, "out"), + config_path = config_path, + to_addr = config.get(case_name, "to"), + port = config.getint("relay", "port")) - sleep(DELAY) + # sleep(DELAY) diff --git a/test/keyhome/crls.d/DIR.txt b/test/keyhome/crls.d/DIR.txt new file mode 100644 index 0000000..2a29a47 --- /dev/null +++ b/test/keyhome/crls.d/DIR.txt @@ -0,0 +1 @@ +v:1: diff --git a/test/keyhome/pubring.kbx b/test/keyhome/pubring.kbx new file mode 100644 index 0000000000000000000000000000000000000000..83e1be16f71dfa5c39e580f11efc92404693ebe4 GIT binary patch literal 3088 zcmb7G2Q-}97XJS+>WC5DkZ3_f3Zl1Qln`Zf6GRv?K@dGei<0QB9wK^3Bcrz+gRxx z+Yzb+vi?J8Qz0M#CF$`1K(hrv0B-I<4Y34TOuy`6n0dP5_9xi@^%0Vvm%P8|0UYmU zNEx4l%7z8EuW|^?<}Z@Wiew*2AOJmSx=7OC-@gIqdD6=MU8nf|mmlVzX4}~HB(f<3 zjDy}B#bko?Kod75hC&~Mi`EJ}5smy*oqt_Q&Ch=ClK}j4W*El~AY9SK7vuiBO|HQQ z!O&Koi3ImR8&n9qrC%Z-=~~LR)TA20 z^_k62m1_guO$(>SGH(`_rmu0T;rkLcCMnIUtb8gg@~xlCe}s3ankP=ApHJwJj4voH zu!q_x_+NRND#;0P*~(Y0vZ1Se;KJemjx#qay`uflIYekZ@4oFL=DV?PcLj-QF}Iiz z>R&JJd%rS*R$O6Af=6l129&-$xa^Ugz&Ww5HUR96qi?+U8m!`F-|7@`C&D#vS(xtY zJNJ$c{o&nlZV5xIozDK1sZhv+miW-ZH7U%{qJ=}|9(zr33)Wg~G|qA;kf1m7kJwmo z`7(~pl$RLlSEj-0#%1OPvyJPXW=H7uyPsrqqIr6dSt7|e&x~zJRi+Zx)$18yOSCx) zZVwH);`eVLwXb;-5k=V}RXhGqbBGc{bfe%^3Bv^7BnW^wQZ})7vEYw%Rl;rWIAR=B z>>WMtx_REQwfmPF^1~ek4nvti8bFzi5BgUSqJ)rB3*n-mq@`%Z0+cXX8fpj?D;+Hj z3<70?P=JurAcQ>xprd6xMjiSd?2yk?XJ%x(Djp@rcd{YH$(K*BsNKOBrc?nNZ1nCr zND;^SQ2Z9h#+X>g)iIzjk14g-SWzSp$9NcDd{cSP(g05~`MTv=$Yqx^?GvcJ!=~1K z)};k6EI`TNS`sJyu5d%UI4)+XH8NPWW!lsFNJCIyZMUFaK~-g-;9YmMig+9_vBJ065ZUZtvdU!@y-A*VNl1polRZI?GL1315}=)znS6 zz$p{J^H5nx^pc~EqP0#^86Fdc4;Rt#K4l-bt|ChPWKkC#rqyDH3W$1hd6Esq?&jD# z#u{Wm$1jq*gW!;y#!Y1U>A0`DUD58M&B%QEVWNKZA3^u?L=Fh9!P#Y+ekx`z<3!E0mBcl`{jVrGupV2kpj-D&X%CI}NYxE}N(MuH$_Ak`hqNV55tiGL;RzP@b z@9WyVR5>?>=&rf?8TsV;bt!)~`nLhk;J(Qr66=93dm&6Ug6OVjrwVQy6+l7408q)U zO3ps7;!hw@{<|n36}#%+LqRF*kTBk<7m>{3O(hIoKSNiS>427}G46gA<4!jOZK zSH5FQQ9S2)TOQiir9KwM9*$sJv+FVQ_pC z%CTNg60KV;J{9fZzxjN{)WLBM@mWsX-ib9uz~{}@B%|ixB=TFRZ;$U$ZlAV4@{d`{ z&#^us5SCX;V)e=+e~z=WMC1 z6*q_InU?a#luTe?>iBfPp{05Ynwrr*p*V?quVWFGag@D}UspKO56GeQ%`EXT=POT8 zIouq-$lz?~(cilnc>dXin<_ran_F|2%ai@j2#tm>8>CdK2^|KVSv=X__3o>c4Qm#U zyFk@APj2G2Qh^}o?djS%^u0qzfz?nBQq$%AA5EA3-`i?YH0+RHCuAUC&^N^ii_0jv zZvS{)#EH7=eHvQCz)$}LXSb`+fnOkwM_^0$qu|0-)snn%;lT%AY5}HHl$VEkgqI#- zR1p@k_ImjNZQjx5YkgZ@K(I}gP<}!4U|h?6^Q?~e7PHGs!z{5U8Ks9X5H`k*??9rMTn9w`yz&&^UaH*2-4p~k{|OX!aaF|h<$e~Q^gV-@-@ z7`nKizVPg~83ozM)p%Z(v`VIOODflEr=;5EuNVq!4c!zs3iKKy)QzGdwflOa<;;hl zw5v`K4{l{NxC!e!G|G9UO=6>8-q@0&E13C>7T!TycQVwt9}y`Me;aW|V{n?Yw)lwN zjtf<2Wsj7}Ss8TO;Sci^h^oHY03N(ivH*TLT8|bMz|JuDo}}X^1i9Rh!RX>xJLN|f z^xkma&c?cT8arENJH`fYY$vkER(_;>7jRqu7wyeG<#v3~qWTt3=j_s}!=9WygEYz8 z%AN{+2JW=-#m5CT&-l=!c}&DQYmQugn$~LP zd;hQD!;NWMIb#gs4Oo(&m%OPgFNW{jHFZgK$S41Lsn9%@>dzlZ=3`O@l1UlRMj8mx zAd@sQ-TM!*9Ue#~8y{bW+77oTveAlCflzQv3l4(RDr2Lku3Xox@x3&8mu)R(4?yov zV>p}7%$Db%?tL*xO8SW@Ak^IKcs1SJ>})-F746*Y{^SflJr|K_P-eCtoZ+u-NC_by z<^Osuj)TIA1rC*Ja3-U{%@*gbPp*fgv~WioNSEo0>=_^nySd_Ci`jv|uR`x;`2vXP z^zjca24Aq<$H*Ft(j000o@T%RnGg%5Jx))pEIAlcOmyFak!}i7lA~d0tV6)OMhsbN zcFs-YtTn&nN>r&7X{(LaGOSq4jyI+NX{aObK{-e_BNyg>z1b9m>w94ypIz?&_y*%R zv~$f3-Q^ek9AW8=B@Ukr8olH#ek%TsjjvHoP(JVterHl%wq=C{IdCFEX8sv&Ahc?{ n1kw`#W;+nQbo*e(XZLpYJ=?kHEAh#9)w%N*ZzVKqsq#Mo@KzdD literal 0 HcmV?d00001 diff --git a/test/keyhome/random_seed b/test/keyhome/random_seed new file mode 100644 index 0000000000000000000000000000000000000000..960176317c5bd65a613745e2a1e40002c132c17a GIT binary patch literal 600 zcmV-e0;m0x7F#IZ%)U+ZOVphWztfSW%lqKz4IZtwX4T{y12-_@Hyy%MB#xsB_C0V%YsYhNnj25XQn? z6|qjVS~1x#705R9-Igwmkt|E=T7)XY8>Uw32WdzC=xx{=dpd*!SOx-$&n<$BaFazd zNbBP(a5{d0t1ZsHIaU(E#1Xh&|C|;aaUVykp2cNVVcJpXXwn`20BcUhV`Bi^1k+#( z|M3=TpA_`n)e8gXwrFmvS!VnHWYU#^0EGLrsV!?eQo|BlG7lr-B0Q|Tzlwe;r%fzE zEgFvGr{_Xm_sL^Ct-6RUmaT_{@3xs2#{^IvcR(V3RNvO7O=x-opoFLNA9!#(+1cii zf4d2*SgC;=5IZ*{QozD^?l~=lf2>j6MY3IY&!0b||@`^B&VW`#{a`OQxPY-eDG7K^iRr06BdDisnY@ zV-L@wph$J_P3XvZ;I>pdDh);p|M*s#9J{lB4NU5S*6#2}HSAe5VuL$tPmD0;XQD|0u0ft|&FQRsQ mc35E8&ZulJ|9348SS32JwKm?Ha2J`1dz3JZ^b0M<=Oib=*CGJ` literal 0 HcmV?d00001 diff --git a/test/keyhome/tofu.db b/test/keyhome/tofu.db new file mode 100644 index 0000000000000000000000000000000000000000..16104b58d2fd2658e1693b733251d6a74c9db7f1 GIT binary patch literal 49152 zcmeI&!Ef4D9KdlqZ6P5^*-k!GXul{ItXf+$wo?x)OT7%05=x2GoGc2D$SRl=o2ne9 zDb@Nv?7shD(k}bUcGxr6!Css7T3KH!i0Aj7f6wpp`w7W%o21od-{_eJGyZ@J<;Fs_@@rj`1#9evePx6^kjB4Zs+%`fME5J}&C z(RV~m2mT~!FG4l#Wnw4DgG6UCPu4y}`ed(OF+m1*!3HH?w<+62fV7_au)(QQKU=S_l zD%`ZTbE|Cn<^NXE)4Y*?p~AE%WH$FNwD-CY;^FLlIJvvhH>vAsI+!ozm0DgeL5I_I{1Q0*~0R#|0009ILD82yi a|BIhvc8CB12q1s}0tg_000IagfWW^;-obqU literal 0 HcmV?d00001 diff --git a/test/keyhome/trustdb.gpg b/test/keyhome/trustdb.gpg new file mode 100644 index 0000000000000000000000000000000000000000..769130320fc347ef0dcc957040c55c75dea82098 GIT binary patch literal 1200 zcmZQfFGy!*W@Ke#Vqi$@G5gAZ9WZiX7sn7CRfiEIV1dza84VXu2#lr!%F+P Date: Fri, 7 Jan 2022 12:00:50 +0100 Subject: [PATCH 07/27] Add a test overview, extract constants To let the user know that tests produce logs, include a message at the end of the test output informing about locations of E2E and Mailgate logs. Also, extract some constants. --- test/e2e_test.py | 53 ++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/test/e2e_test.py b/test/e2e_test.py index c730144..68044d4 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -1,5 +1,24 @@ #!/usr/local/bin/python2 +# +# gpg-mailgate +# +# This file is part of the gpg-mailgate source code. +# +# gpg-mailgate is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# gpg-mailgate source code is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with gpg-mailgate source code. If not, see . +# + import os import sys @@ -10,13 +29,12 @@ import logging from time import sleep -EOL = "\n" - -DELAY = 3 - RELAY_SCRIPT = "test/relay.py" CONFIG_FILE = "test/gpg-mailgate.conf" +KEY_HOME = "test/keyhome" +CERT_HOME = "test/certs" + PYTHON_BIN = "python2.7" def build_config(config): @@ -61,17 +79,6 @@ def load_file(name): return contents -def strip_eols(strings): - return map(lambda s: s.strip("\r"), strings) - -def compare(result, expected): - result_lines = strip_eols(result.split(EOL)) - expected_lines = strip_eols(expected.split(EOL)) - - return difflib.unified_diff(expected_lines, result_lines, - fromfile='expected', - tofile='output') - def report_result(message_file, expected, test_output): status = None if expected in test_output: @@ -95,10 +102,10 @@ def execute_e2e_test(message_file, expected, **kwargs): message_file) result_command = "%s %s %d" % (PYTHON_BIN, RELAY_SCRIPT, kwargs["port"]) - logging.debug("Spawning: '%s'" % (result_command)) + logging.debug("Spawning relay: '%s'" % (result_command)) pipe = os.popen(result_command, 'r') - logging.debug("Spawning: '%s'" % (test_command)) + logging.debug("Spawning GPG-Lacre: '%s'" % (test_command)) msgin = os.popen(test_command, 'w') msgin.write(load_file(message_file)) msgin.close() @@ -118,8 +125,10 @@ def load_test_config(): config = load_test_config() +log_paths = {"e2e": "test/logs/e2e.log", + "lacre": "test/logs/gpg-mailgate.log"} -logging.basicConfig(filename = "test/logs/e2e.log", +logging.basicConfig(filename = log_paths["e2e"], format = "%(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s", datefmt = "%Y-%m-%d %H:%M:%S", level = logging.DEBUG) @@ -128,9 +137,9 @@ config_path = os.getcwd() + "/" + CONFIG_FILE write_test_config(config_path, port = config.getint("relay", "port"), - gpg_keyhome = "test/keyhome", - smime_certpath = "test/certs", - log_file = "test/logs/gpg-mailgate.log") + gpg_keyhome = KEY_HOME, + smime_certpath = CERT_HOME, + log_file = log_paths["lacre"]) for case_no in range(1, config.getint("tests", "cases")+1): case_name = "case-%d" % (case_no) @@ -142,4 +151,4 @@ for case_no in range(1, config.getint("tests", "cases")+1): to_addr = config.get(case_name, "to"), port = config.getint("relay", "port")) - # sleep(DELAY) +print "See diagnostic output for details. Tests: '%s', Lacre: '%s'" % (log_paths["e2e"], log_paths["lacre"]) From 01377f4dd2038f78e313cfee20d3457bef62135a Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Fri, 7 Jan 2022 12:03:38 +0100 Subject: [PATCH 08/27] Keep test/certs directory --- test/certs/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/certs/.keep diff --git a/test/certs/.keep b/test/certs/.keep new file mode 100644 index 0000000..e69de29 From e90a29c9ff8fec78ee7a0c022f39c02027cb5600 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Fri, 7 Jan 2022 12:04:14 +0100 Subject: [PATCH 09/27] Ignore temporary test directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6b8344e..3cafdc4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ nosetests.xml # GPG-Mailgate test files test/logs +test/tmp test/gpg-mailgate.conf # Emacs files From 98c4580775f949b5857670f1eead82b5083f0e9f Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Fri, 7 Jan 2022 12:10:30 +0100 Subject: [PATCH 10/27] Document E2E tests --- doc/testing.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 doc/testing.md diff --git a/doc/testing.md b/doc/testing.md new file mode 100644 index 0000000..1d84099 --- /dev/null +++ b/doc/testing.md @@ -0,0 +1,31 @@ +# Testing + +First tests have been set up to cover GPG Mailgate with at least basic test +that would be easy to run. The tests are called "end-to-end", meaning that we +feed some input to GPG Mailgate and inspect the output. + +## Running tests + +To run tests, use command `make test`. + +Tests produce some helpful logs, so inspect contents of `test/logs` directory +if something goes wrong. + +## Key building blocks + +- *Test Script* (`test/e2e_test.py`) that orchestrates the other components. + It performs test cases described in the *Test Configuration*. It spawns + *Test Mail Relay* and *GPG Mailgate* in appropriate order. +- *Test Mail Relay* (`test/relay.py`), a simplistic mail daemon that only + supports the happy path. It accepts a mail message and prints it to + stdandard output. +- *Test Configuration* (`test/e2e.ini`) specifies test cases: their input, + expected results and helpful documentation. It also specifies the port that + the *Test Mail Relay* should listen on. + +## Limitations + +Currently tests only check if the message has been encrypted, without +verifying that the correct key has been used. That's because we don't know +(yet) how to have a reproducible encrypted message. Option +`--faked-system-time` wasn't enough to produce identical output. From fc2779ef7d0dd2aca5a98480e478d1d1e6785f70 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sat, 8 Jan 2022 13:42:23 +0100 Subject: [PATCH 11/27] Improve test code structure - Move things to configuration file where appropriate (logging format, etc.). - Rework execute_e2e_test signature to simplify it and get rid of keyword arguments. - Simplify output. - Include a header comment in configuration file. --- test/e2e.ini | 60 +++++++++++++++++++++++++++++++++++++----------- test/e2e_test.py | 57 +++++++++++++++++++++------------------------ 2 files changed, 72 insertions(+), 45 deletions(-) diff --git a/test/e2e.ini b/test/e2e.ini index b65d0dc..2d6fb0b 100644 --- a/test/e2e.ini +++ b/test/e2e.ini @@ -1,23 +1,55 @@ +# +# gpg-mailgate +# +# This file is part of the gpg-mailgate source code. +# +# gpg-mailgate is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# gpg-mailgate source code is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with gpg-mailgate source code. If not, see . +# + +# NOTE: We use : syntax, because some values contain +# colons and that is default ConfigParser key-value separator. + [relay] -port = 2500 +port: 2500 +script: test/relay.py + +[dirs] +keys: test/keyhome +certs: test/certs [tests] -cases = 3 +# Number of "test-*" sections in this file, describing test cases. +cases: 3 +e2e_log: test/logs/e2e.log +e2e_log_format: %(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s +e2e_log_datefmt: %Y-%m-%d %H:%M:%S +lacre_log: test/logs/gpg-mailgate.log [case-1] -descr = Clear text message to a user without a key -to = carlos@disposlab -in = test/msgin/clear2clear.msg -out = Body of the message. +descr: Clear text message to a user without a key +to: carlos@disposlab +in: test/msgin/clear2clear.msg +out: Body of the message. [case-2] -descr = Clear text message to a user with an RSA key -to = alice@disposlab -in = test/msgin/clear2rsa.msg -out = -----BEGIN PGP MESSAGE----- +descr: Clear text message to a user with an RSA key +to: alice@disposlab +in: test/msgin/clear2rsa.msg +out: -----BEGIN PGP MESSAGE----- [case-3] -descr = Clear text message to a user with an Ed25519 key -to = bob@disposlab -in = test/msgin/clear2ed.msg -out = -----BEGIN PGP MESSAGE----- +descr: Clear text message to a user with an Ed25519 key +to: bob@disposlab +in: test/msgin/clear2ed.msg +out: -----BEGIN PGP MESSAGE----- diff --git a/test/e2e_test.py b/test/e2e_test.py index 68044d4..d53d6d9 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -32,9 +32,6 @@ from time import sleep RELAY_SCRIPT = "test/relay.py" CONFIG_FILE = "test/gpg-mailgate.conf" -KEY_HOME = "test/keyhome" -CERT_HOME = "test/certs" - PYTHON_BIN = "python2.7" def build_config(config): @@ -86,28 +83,29 @@ def report_result(message_file, expected, test_output): else: status = "Failure" - print "%s %s" % (message_file.ljust(30, '.'), status) + print message_file.ljust(30), status -def frozen_time_expr(timestamp): - if timestamp is None: - return "" - else: - return "GPG_FROZEN_TIME=%s" % (timestamp) +def execute_e2e_test(case_name, config, config_path): + """Read test case configuration from config and run that test case. + + Parameter case_name should refer to a section in test + config file. Each of these sections should contain + following properties: 'descr', 'to', 'in' and 'out'. + """ -def execute_e2e_test(message_file, expected, **kwargs): test_command = "GPG_MAILGATE_CONFIG=%s %s gpg-mailgate.py %s < %s" % ( - kwargs["config_path"], + config_path, PYTHON_BIN, - kwargs["to_addr"], - message_file) - result_command = "%s %s %d" % (PYTHON_BIN, RELAY_SCRIPT, kwargs["port"]) + config.get(case_name, "to"), + config.get(case_name, "in")) + result_command = "%s %s %d" % (PYTHON_BIN, config.get("relay", "script"), config.getint("relay", "port")) logging.debug("Spawning relay: '%s'" % (result_command)) pipe = os.popen(result_command, 'r') logging.debug("Spawning GPG-Lacre: '%s'" % (test_command)) msgin = os.popen(test_command, 'w') - msgin.write(load_file(message_file)) + msgin.write(load_file(config.get(case_name, "in"))) msgin.close() testout = pipe.read() @@ -115,7 +113,7 @@ def execute_e2e_test(message_file, expected, **kwargs): logging.debug("Read %d characters of test output: '%s'" % (len(testout), testout)) - report_result(message_file, expected, testout) + report_result(config.get(case_name, "in"), config.get(case_name, "out"), testout) def load_test_config(): cp = ConfigParser.ConfigParser() @@ -125,30 +123,27 @@ def load_test_config(): config = load_test_config() -log_paths = {"e2e": "test/logs/e2e.log", - "lacre": "test/logs/gpg-mailgate.log"} -logging.basicConfig(filename = log_paths["e2e"], - format = "%(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s", - datefmt = "%Y-%m-%d %H:%M:%S", +logging.basicConfig(filename = config.get("tests", "e2e_log"), + # Get raw values of log and date formats because they + # contain %-sequences and we don't want them to be expanded + # by the ConfigParser. + format = config.get("tests", "e2e_log_format", True), + datefmt = config.get("tests", "e2e_log_datefmt", True), level = logging.DEBUG) config_path = os.getcwd() + "/" + CONFIG_FILE write_test_config(config_path, port = config.getint("relay", "port"), - gpg_keyhome = KEY_HOME, - smime_certpath = CERT_HOME, - log_file = log_paths["lacre"]) + gpg_keyhome = config.get("dirs", "keys"), + smime_certpath = config.get("dirs", "certs"), + log_file = config.get("tests", "lacre_log")) for case_no in range(1, config.getint("tests", "cases")+1): case_name = "case-%d" % (case_no) - print "Executing: %s" % (config.get(case_name, "descr")) + logging.info("Executing %s: %s", case_name, config.get(case_name, "descr")) - execute_e2e_test(config.get(case_name, "in"), - config.get(case_name, "out"), - config_path = config_path, - to_addr = config.get(case_name, "to"), - port = config.getint("relay", "port")) + execute_e2e_test(case_name, config, config_path) -print "See diagnostic output for details. Tests: '%s', Lacre: '%s'" % (log_paths["e2e"], log_paths["lacre"]) +print "See diagnostic output for details. Tests: '%s', Lacre: '%s'" % (config.get("tests", "e2e_log"), config.get("tests", "lacre_log")) From 2cf60dec40d00de7c8b7b08a8abe421cb7c8af7d Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sun, 9 Jan 2022 21:00:51 +0100 Subject: [PATCH 12/27] Add unit tests for GnuPG command-line generator Extract a function to calculate GPG commands to be executed and cover it with unit tests. --- GnuPG/__init__.py | 20 +++++++++++--------- Makefile | 13 +++++++++++++ test/test_gnupg.py | 16 ++++++++++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 test/test_gnupg.py diff --git a/GnuPG/__init__.py b/GnuPG/__init__.py index 294f8c9..540622e 100644 --- a/GnuPG/__init__.py +++ b/GnuPG/__init__.py @@ -31,8 +31,12 @@ LINE_USER_ID = 'uid' POS_FINGERPRINT = 9 +def build_command(key_home, *args, **kwargs): + cmd = ["gpg", '--homedir', key_home] + list(args) + return cmd + def private_keys( keyhome ): - cmd = ['/usr/bin/gpg', '--homedir', keyhome, '--list-secret-keys', '--with-colons'] + cmd = build_command(keyhome, '--list-secret-keys', '--with-colons') p = subprocess.Popen( cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.wait() keys = dict() @@ -46,7 +50,7 @@ def private_keys( keyhome ): return keys def public_keys( keyhome ): - cmd = ['/usr/bin/gpg', '--homedir', keyhome, '--list-keys', '--with-colons'] + cmd = build_command(keyhome, '--list-keys', '--with-colons') p = subprocess.Popen( cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.wait() @@ -78,7 +82,7 @@ def confirm_key( content, email ): os.mkdir(tmpkeyhome) localized_env = os.environ.copy() localized_env["LANG"] = "C" - p = subprocess.Popen( ['/usr/bin/gpg', '--homedir', tmpkeyhome, '--import', '--batch'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=localized_env ) + p = subprocess.Popen( build_command(tmpkeyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=localized_env ) result = p.communicate(input=content)[1] confirmed = False @@ -97,7 +101,7 @@ def confirm_key( content, email ): # adds a key and ensures it has the given email address def add_key( keyhome, content ): - p = subprocess.Popen( ['/usr/bin/gpg', '--homedir', keyhome, '--import', '--batch'], stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE ) + p = subprocess.Popen( build_command(keyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.communicate(input=content) p.wait() @@ -107,7 +111,7 @@ def delete_key( keyhome, email ): if result[1]: # delete all keys matching this email address - p = subprocess.Popen( ['/usr/bin/gpg', '--homedir', keyhome, '--delete-key', '--batch', '--yes', result[1]], stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + p = subprocess.Popen( build_command(keyhome, '--delete-key', '--batch', '--yes', result[1]), stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.wait() return True @@ -131,7 +135,7 @@ class GPGEncryptor: return (encdata, p.returncode) def _command(self): - cmd = ["/usr/bin/gpg", "--trust-model", "always", "--homedir", self._keyhome, "--batch", "--yes", "--pgp7", "--no-secmem-warning", "-a", "-e"] + cmd = build_command(self._keyhome, "--trust-model", "always", "--batch", "--yes", "--pgp7", "--no-secmem-warning", "-a", "-e") # add recipients for recipient in self._recipients: @@ -159,6 +163,4 @@ class GPGDecryptor: return (decdata, p.returncode) def _command(self): - cmd = ["/usr/bin/gpg", "--trust-model", "always", "--homedir", self._keyhome, "--batch", "--yes", "--no-secmem-warning", "-a", "-d"] - - return cmd + return build_command(self._keyhome, "--trust-model", "always", "--batch", "--yes", "--no-secmem-warning", "-a", "-d") diff --git a/Makefile b/Makefile index 629a04b..a14332a 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,22 @@ PYTHON = python2.7 .PHONY: test pre-clean clean +# +# Run a set of end-to-end tests. +# +# Test scenarios are described and configured by the test/e2e.ini +# file. Basically this is just a script that feeds GPG Mailgate with +# known input and checks whether output meets expectations. +# test: test/tmp test/logs pre-clean $(PYTHON) test/e2e_test.py +# +# Run unit tests +# +unittest: + $(PYTHON) -m unittest discover -s test + pre-clean: rm -fv test/gpg-mailgate.conf rm -f test/logs/*.log diff --git a/test/test_gnupg.py b/test/test_gnupg.py new file mode 100644 index 0000000..5712acd --- /dev/null +++ b/test/test_gnupg.py @@ -0,0 +1,16 @@ +import GnuPG + +import unittest + +class GnuPGUtilitiesTest(unittest.TestCase): + def test_build_default_command(self): + cmd = GnuPG.build_command("test/keyhome") + self.assertEqual(cmd, ["gpg", "--homedir", "test/keyhome"]) + + def test_build_command_extended_with_args(self): + cmd = GnuPG.build_command("test/keyhome", "--foo", "--bar") + self.assertEqual(cmd, ["gpg", "--homedir", "test/keyhome", "--foo", "--bar"]) + + +if __name__ == '__main__': + unittest.main() From 1002da78fa2c309842f734cca89c839fb57a53dc Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sun, 9 Jan 2022 21:58:14 +0100 Subject: [PATCH 13/27] Clean up test messages Remove unused msgout files, fix addresses in msgin files. --- doc/testing.md | 2 +- test/msgin/clear2clear.msg | 4 ++-- test/msgin/clear2ed.msg | 4 ++-- test/msgin/clear2rsa.msg | 2 +- test/msgout/clear2clear.msg | 5 ----- test/msgout/clear2ed.msg | 5 ----- test/msgout/clear2rsa.msg | 5 ----- 7 files changed, 6 insertions(+), 21 deletions(-) delete mode 100644 test/msgout/clear2clear.msg delete mode 100644 test/msgout/clear2ed.msg delete mode 100644 test/msgout/clear2rsa.msg diff --git a/doc/testing.md b/doc/testing.md index 1d84099..a462b0f 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -6,7 +6,7 @@ feed some input to GPG Mailgate and inspect the output. ## Running tests -To run tests, use command `make test`. +To run tests, use command `make test` or `make unittest`. Tests produce some helpful logs, so inspect contents of `test/logs` directory if something goes wrong. diff --git a/test/msgin/clear2clear.msg b/test/msgin/clear2clear.msg index 5fe7ff7..ae577d1 100644 --- a/test/msgin/clear2clear.msg +++ b/test/msgin/clear2clear.msg @@ -1,5 +1,5 @@ -From: Bob -To: Alice +From: Dave +To: Carlos Subject: Test Body of the message. diff --git a/test/msgin/clear2ed.msg b/test/msgin/clear2ed.msg index 5fe7ff7..63b6c66 100644 --- a/test/msgin/clear2ed.msg +++ b/test/msgin/clear2ed.msg @@ -1,5 +1,5 @@ -From: Bob -To: Alice +From: Dave +To: Bob Subject: Test Body of the message. diff --git a/test/msgin/clear2rsa.msg b/test/msgin/clear2rsa.msg index 5fe7ff7..9dfd134 100644 --- a/test/msgin/clear2rsa.msg +++ b/test/msgin/clear2rsa.msg @@ -1,4 +1,4 @@ -From: Bob +From: Dave To: Alice Subject: Test diff --git a/test/msgout/clear2clear.msg b/test/msgout/clear2clear.msg deleted file mode 100644 index 5fe7ff7..0000000 --- a/test/msgout/clear2clear.msg +++ /dev/null @@ -1,5 +0,0 @@ -From: Bob -To: Alice -Subject: Test - -Body of the message. diff --git a/test/msgout/clear2ed.msg b/test/msgout/clear2ed.msg deleted file mode 100644 index 5fe7ff7..0000000 --- a/test/msgout/clear2ed.msg +++ /dev/null @@ -1,5 +0,0 @@ -From: Bob -To: Alice -Subject: Test - -Body of the message. diff --git a/test/msgout/clear2rsa.msg b/test/msgout/clear2rsa.msg deleted file mode 100644 index 5fe7ff7..0000000 --- a/test/msgout/clear2rsa.msg +++ /dev/null @@ -1,5 +0,0 @@ -From: Bob -To: Alice -Subject: Test - -Body of the message. From 3b9f714cdb2c340dac8c15f229121f3863be6bbb Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Mon, 10 Jan 2022 17:47:23 +0100 Subject: [PATCH 14/27] Ignore random_seed Do not keep test/keyhome/random_seed in repository. --- .gitignore | 1 + test/keyhome/random_seed | Bin 600 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 test/keyhome/random_seed diff --git a/.gitignore b/.gitignore index 3cafdc4..140af95 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ nosetests.xml test/logs test/tmp test/gpg-mailgate.conf +test/keyhome/random_seed # Emacs files *~ diff --git a/test/keyhome/random_seed b/test/keyhome/random_seed deleted file mode 100644 index 960176317c5bd65a613745e2a1e40002c132c17a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 600 zcmV-e0;m0x7F#IZ%)U+ZOVphWztfSW%lqKz4IZtwX4T{y12-_@Hyy%MB#xsB_C0V%YsYhNnj25XQn? z6|qjVS~1x#705R9-Igwmkt|E=T7)XY8>Uw32WdzC=xx{=dpd*!SOx-$&n<$BaFazd zNbBP(a5{d0t1ZsHIaU(E#1Xh&|C|;aaUVykp2cNVVcJpXXwn`20BcUhV`Bi^1k+#( z|M3=TpA_`n)e8gXwrFmvS!VnHWYU#^0EGLrsV!?eQo|BlG7lr-B0Q|Tzlwe;r%fzE zEgFvGr{_Xm_sL^Ct-6RUmaT_{@3xs2#{^IvcR(V3RNvO7O=x-opoFLNA9!#(+1cii zf4d2*SgC;=5IZ*{QozD^?l~=lf2>j6MY3IY&!0b||@`^B&VW`#{a`OQxPY-eDG7K^iRr06BdDisnY@ zV-L@wph$J_P3XvZ;I>pdDh);p|M*s#9J{lB4NU5S*6#2}HSAe5VuL$tPmD0;XQD|0u0ft|&FQRsQ mc35E8&ZulJ|9348SS32JwKm?Ha2J`1dz3JZ^b0M<=Oib=*CGJ` From 5f02223ec7d8bf7fb631b8128e5befafe1f34fae Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sat, 8 Jan 2022 14:06:18 +0100 Subject: [PATCH 15/27] Perform automatic migration to Python 3.x Use lib2to3 automatic migration tool provided by Python 2.x to convert codebase to new idioms. Command line: find . -type f -name '*.py' \ -exec python2.7 -m lib2to3 \ -f all -f idioms -f buffer -f set_literal -f ws_comma -w \ '{}' '+' --- GnuPG/__init__.py | 6 +++--- gpg-mailgate-web/cron.py | 12 ++++++------ gpg-mailgate.py | 38 +++++++++++++++++++------------------- register-handler.py | 12 ++++++------ test/e2e_test.py | 10 +++++----- test/relay.py | 8 ++++---- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/GnuPG/__init__.py b/GnuPG/__init__.py index 540622e..6404720 100644 --- a/GnuPG/__init__.py +++ b/GnuPG/__init__.py @@ -101,7 +101,7 @@ def confirm_key( content, email ): # adds a key and ensures it has the given email address def add_key( keyhome, content ): - p = subprocess.Popen( build_command(keyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + p = subprocess.Popen( ['/usr/bin/gpg', '--homedir', keyhome, '--import', '--batch'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.communicate(input=content) p.wait() @@ -130,7 +130,7 @@ class GPGEncryptor: self._message += message def encrypt(self): - p = subprocess.Popen( self._command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE ) + p = subprocess.Popen( self._command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) encdata = p.communicate(input=self._message)[0] return (encdata, p.returncode) @@ -158,7 +158,7 @@ class GPGDecryptor: self._message += message def decrypt(self): - p = subprocess.Popen( self._command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE ) + p = subprocess.Popen( self._command(), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) decdata = p.communicate(input=self._message)[0] return (decdata, p.returncode) diff --git a/gpg-mailgate-web/cron.py b/gpg-mailgate-web/cron.py index 3f305ba..98cd8d1 100644 --- a/gpg-mailgate-web/cron.py +++ b/gpg-mailgate-web/cron.py @@ -19,7 +19,7 @@ # along with gpg-mailgate source code. If not, see . # -from ConfigParser import RawConfigParser +from configparser import RawConfigParser import GnuPG import MySQLdb import smtplib @@ -64,7 +64,7 @@ for sect in _cfg.sections(): for (name, value) in _cfg.items(sect): cfg[sect][name] = value -if cfg.has_key('database') and cfg['database'].has_key('enabled') and cfg['database']['enabled'] == 'yes' and cfg['database'].has_key('name') and cfg['database'].has_key('host') and cfg['database'].has_key('username') and cfg['database'].has_key('password'): +if 'database' in cfg and 'enabled' in cfg['database'] and cfg['database']['enabled'] == 'yes' and 'name' in cfg['database'] and 'host' in cfg['database'] and 'username' in cfg['database'] and 'password' in cfg['database']: connection = MySQLdb.connect(host = cfg['database']['host'], user = cfg['database']['username'], passwd = cfg['database']['password'], db = cfg['database']['name'], port = 3306) cursor = connection.cursor() @@ -83,17 +83,17 @@ if cfg.has_key('database') and cfg['database'].has_key('enabled') and cfg['datab GnuPG.add_key(cfg['gpg']['keyhome'], row[0]) # import the key to gpg cursor.execute("UPDATE gpgmw_keys SET status = 1 WHERE id = %s", (row[1],)) # mark key as accepted appendLog('Imported key from <' + row[2] + '>') - if cfg['cron'].has_key('send_email') and cfg['cron']['send_email'] == 'yes': + if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key registration successful", "registrationSuccess.md", row[2] ) else: cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) # delete key appendLog('Import confirmation failed for <' + row[2] + '>') - if cfg['cron'].has_key('send_email') and cfg['cron']['send_email'] == 'yes': + if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key registration failed", "registrationError.md", row[2] ) else: # delete key so we don't continue processing it cursor.execute("DELETE FROM gpgmw_keys WHERE id = %s", (row[1],)) - if cfg['cron'].has_key('send_email') and cfg['cron']['send_email'] == 'yes': + if 'send_email' in cfg['cron'] and cfg['cron']['send_email'] == 'yes': send_msg( "PGP key deleted", "keyDeleted.md", row[2]) connection.commit() @@ -108,4 +108,4 @@ if cfg.has_key('database') and cfg['database'].has_key('enabled') and cfg['datab appendLog('Deleted key for <' + row[0] + '>') connection.commit() else: - print "Warning: doing nothing since database settings are not configured!" + print("Warning: doing nothing since database settings are not configured!") diff --git a/gpg-mailgate.py b/gpg-mailgate.py index b39c786..e520aa5 100755 --- a/gpg-mailgate.py +++ b/gpg-mailgate.py @@ -19,7 +19,7 @@ # along with gpg-mailgate source code. If not, see . # -from ConfigParser import RawConfigParser +from configparser import RawConfigParser from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart import copy @@ -94,7 +94,7 @@ def gpg_decrypt( raw_message, recipients ): keys[fingerprint] = sanitize_case_sense(keys[fingerprint]) for to in recipients: - if to in keys.values() and not get_bool_from_cfg('default', 'dec_keymap_only', 'yes'): + if to in list(keys.values()) and not get_bool_from_cfg('default', 'dec_keymap_only', 'yes'): gpg_to.append(to) # Is this recipient defined in regex for default decryption? elif not (dec_regex is None) and not (re.match(dec_regex, to) is None): @@ -106,7 +106,7 @@ def gpg_decrypt( raw_message, recipients ): if not cfg['dec_keymap'][to] in keys: log("Key '%s' in decryption keymap not found in keyring for email address '%s'. Won't decrypt." % (cfg['dec_keymap'][to], to)) # Avoid unwanted encryption if set - if to in keys.values() and get_bool_from_cfg('default', 'failsave_dec', 'yes'): + if to in list(keys.values()) and get_bool_from_cfg('default', 'failsave_dec', 'yes'): noenc_to.append(to) else: ungpg_to.append(to) @@ -116,7 +116,7 @@ def gpg_decrypt( raw_message, recipients ): if verbose: log("Recipient (%s) not in PGP domain list for decrypting." % to) # Avoid unwanted encryption if set - if to in keys.values() and get_bool_from_cfg('default', 'failsave_dec', 'yes'): + if to in list(keys.values()) and get_bool_from_cfg('default', 'failsave_dec', 'yes'): noenc_to.append(to) else: ungpg_to.append(to) @@ -206,7 +206,7 @@ def decrypt_inline_with_attachments( payloads, success, message = None ): message = email.mime.multipart.MIMEMultipart(payloads.get_content_subtype()) for payload in payloads.get_payload(): - if( type( payload.get_payload() ) == list ): + if( isinstance(payload.get_payload(), list) ): # Take care of cascaded MIME messages submessage, subsuccess = decrypt_inline_with_attachments( payload, success ) message.attach(submessage) @@ -317,7 +317,7 @@ def gpg_encrypt( raw_message, recipients ): log("Key '%s' in encrypt keymap not found in keyring for email address '%s'." % (cfg['enc_keymap'][to], to)) # Check if key in keychain is present - if to in keys.values() and not get_bool_from_cfg('default', 'enc_keymap_only', 'yes'): + if to in list(keys.values()) and not get_bool_from_cfg('default', 'enc_keymap_only', 'yes'): gpg_to.append( (to, to) ) continue @@ -341,7 +341,7 @@ def gpg_encrypt( raw_message, recipients ): ungpg_to.append(to) if gpg_to != list(): - log("Encrypting email to: %s" % ' '.join( map(lambda x: x[0], gpg_to) )) + log("Encrypting email to: %s" % ' '.join( [x[0] for x in gpg_to] )) # Getting PGP style for recipient gpg_to_smtp_mime = list() @@ -378,8 +378,8 @@ def gpg_encrypt( raw_message, recipients ): if get_bool_from_cfg('default', 'add_header', 'yes'): raw_message_mime['X-GPG-Mailgate'] = 'Encrypted by GPG Mailgate' - if raw_message_mime.has_key('Content-Transfer-Encoding'): - raw_message_mime.replace_header('Content-Transfer-Encoding','8BIT') + if 'Content-Transfer-Encoding' in raw_message_mime: + raw_message_mime.replace_header('Content-Transfer-Encoding', '8BIT') else: raw_message_mime['Content-Transfer-Encoding'] = '8BIT' @@ -395,8 +395,8 @@ def gpg_encrypt( raw_message, recipients ): if get_bool_from_cfg('default', 'add_header', 'yes'): raw_message_inline['X-GPG-Mailgate'] = 'Encrypted by GPG Mailgate' - if raw_message_inline.has_key('Content-Transfer-Encoding'): - raw_message_inline.replace_header('Content-Transfer-Encoding','8BIT') + if 'Content-Transfer-Encoding' in raw_message_inline: + raw_message_inline.replace_header('Content-Transfer-Encoding', '8BIT') else: raw_message_inline['Content-Transfer-Encoding'] = '8BIT' @@ -411,11 +411,11 @@ def encrypt_all_payloads_inline( message, gpg_to_cmdline ): # This breaks cascaded MIME messages. Blame PGP/INLINE. encrypted_payloads = list() - if type( message.get_payload() ) == str: + if isinstance(message.get_payload(), str): return encrypt_payload( message, gpg_to_cmdline ).get_payload() for payload in message.get_payload(): - if( type( payload.get_payload() ) == list ): + if( isinstance(payload.get_payload(), list) ): encrypted_payloads.extend( encrypt_all_payloads_inline( payload, gpg_to_cmdline ) ) else: encrypted_payloads.append( encrypt_payload( payload, gpg_to_cmdline ) ) @@ -437,13 +437,13 @@ def encrypt_all_payloads_mime( message, gpg_to_cmdline ): submsg2.set_param('inline', "", 'Content-Disposition' ) submsg2.set_param('filename', "encrypted.asc", 'Content-Disposition' ) - if type ( message.get_payload() ) == str: + if isinstance(message.get_payload(), str): # WTF! It seems to swallow the first line. Not sure why. Perhaps # it's skipping an imaginary blank line someplace. (ie skipping a header) # Workaround it here by prepending a blank line. # This happens only on text only messages. additionalSubHeader="" - if message.has_key('Content-Type') and not message['Content-Type'].startswith('multipart'): + if 'Content-Type' in message and not message['Content-Type'].startswith('multipart'): additionalSubHeader="Content-Type: "+message['Content-Type']+"\n" submsg2.set_payload(additionalSubHeader+"\n" +message.get_payload(decode=True)) check_nested = True @@ -460,7 +460,7 @@ def encrypt_all_payloads_mime( message, gpg_to_cmdline ): boundary = junk_msg.get_boundary() # This also modifies the boundary in the body of the message, ie it gets parsed. - if message.has_key('Content-Type'): + if 'Content-Type' in message: message.replace_header('Content-Type', "multipart/encrypted; protocol=\"application/pgp-encrypted\";\nboundary=\"%s\"\n" % boundary) else: message['Content-Type'] = "multipart/encrypted; protocol=\"application/pgp-encrypted\";\nboundary=\"%s\"\n" % boundary @@ -608,7 +608,7 @@ def generate_message_from_payloads( payloads, message = None ): message = email.mime.multipart.MIMEMultipart(payloads.get_content_subtype()) for payload in payloads.get_payload(): - if( type( payload.get_payload() ) == list ): + if( isinstance(payload.get_payload(), list) ): message.attach(generate_message_from_payloads(payload)) else: message.attach(payload) @@ -624,7 +624,7 @@ def get_first_payload( payloads ): def send_msg( message, recipients ): - recipients = filter(None, recipients) + recipients = [_f for _f in recipients if _f] if recipients: if not (get_bool_from_cfg('relay', 'host') and get_bool_from_cfg('relay', 'port')): log("Missing settings for relay. Sending email aborted.") @@ -632,7 +632,7 @@ def send_msg( message, recipients ): log("Sending email to: <%s>" % '> <'.join( recipients )) relay = (cfg['relay']['host'], int(cfg['relay']['port'])) smtp = smtplib.SMTP(relay[0], relay[1]) - if cfg.has_key('relay') and cfg['relay'].has_key('starttls') and cfg['relay']['starttls'] == 'yes': + if 'relay' in cfg and 'starttls' in cfg['relay'] and cfg['relay']['starttls'] == 'yes': smtp.starttls() smtp.sendmail( from_addr, recipients, message ) else: diff --git a/register-handler.py b/register-handler.py index 5b1cf9a..f4c1e57 100644 --- a/register-handler.py +++ b/register-handler.py @@ -1,6 +1,6 @@ #!/usr/bin/python -from ConfigParser import RawConfigParser +from configparser import RawConfigParser import email, os, smtplib, sys, traceback, markdown, syslog, requests from M2Crypto import BIO, Rand, SMIME, X509 @@ -17,7 +17,7 @@ for sect in _cfg.sections(): cfg[sect][name] = value def log(msg): - if cfg.has_key('logging') and cfg['logging'].has_key('file'): + if 'logging' in cfg and 'file' in cfg['logging']: if cfg['logging']['file'] == "syslog": syslog.syslog(syslog.LOG_INFO | syslog.LOG_MAIL, msg) else: @@ -78,9 +78,9 @@ if __name__ == "__main__": sys.exit(0) if sign_type == 'smime': - raw_sig = sign_part.get_payload().replace("\n","") + raw_sig = sign_part.get_payload().replace("\n", "") # re-wrap signature so that it fits base64 standards - cooked_sig = '\n'.join(raw_sig[pos:pos+76] for pos in xrange(0, len(raw_sig), 76)) + cooked_sig = '\n'.join(raw_sig[pos:pos+76] for pos in range(0, len(raw_sig), 76)) # now, wrap the signature in a PKCS7 block sig = """ @@ -106,7 +106,7 @@ if __name__ == "__main__": # format in user-specific data # sending success mail only for S/MIME as GPGMW handles this on its own success_msg = file(cfg['mailregister']['mail_templates']+"/registrationSuccess.md").read() - success_msg = success_msg.replace("[:FROMADDRESS:]",from_addr) + success_msg = success_msg.replace("[:FROMADDRESS:]", from_addr) msg = MIMEMultipart("alternative") msg["From"] = cfg['mailregister']['register_email'] @@ -128,7 +128,7 @@ if __name__ == "__main__": if r.status_code != 200: log("Could not hand registration over to GPGMW. Error: %s" % r.status_code) error_msg = file(cfg['mailregister']['mail_templates']+"/gpgmwFailed.md").read() - error_msg = error_msg.replace("[:FROMADDRESS:]",from_addr) + error_msg = error_msg.replace("[:FROMADDRESS:]", from_addr) msg = MIMEMultipart("alternative") msg["From"] = cfg['mailregister']['register_email'] diff --git a/test/e2e_test.py b/test/e2e_test.py index d53d6d9..0f7705f 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -24,7 +24,7 @@ import sys import difflib -import ConfigParser +import configparser import logging from time import sleep @@ -35,7 +35,7 @@ CONFIG_FILE = "test/gpg-mailgate.conf" PYTHON_BIN = "python2.7" def build_config(config): - cp = ConfigParser.ConfigParser() + cp = configparser.ConfigParser() cp.add_section("logging") cp.set("logging", "file", config["log_file"]) @@ -83,7 +83,7 @@ def report_result(message_file, expected, test_output): else: status = "Failure" - print message_file.ljust(30), status + print(message_file.ljust(30), status) def execute_e2e_test(case_name, config, config_path): """Read test case configuration from config and run that test case. @@ -116,7 +116,7 @@ def execute_e2e_test(case_name, config, config_path): report_result(config.get(case_name, "in"), config.get(case_name, "out"), testout) def load_test_config(): - cp = ConfigParser.ConfigParser() + cp = configparser.ConfigParser() cp.read("test/e2e.ini") return cp @@ -146,4 +146,4 @@ for case_no in range(1, config.getint("tests", "cases")+1): execute_e2e_test(case_name, config, config_path) -print "See diagnostic output for details. Tests: '%s', Lacre: '%s'" % (config.get("tests", "e2e_log"), config.get("tests", "lacre_log")) +print("See diagnostic output for details. Tests: '%s', Lacre: '%s'" % (config.get("tests", "e2e_log"), config.get("tests", "lacre_log"))) diff --git a/test/relay.py b/test/relay.py index 01b93d4..bc6969c 100644 --- a/test/relay.py +++ b/test/relay.py @@ -40,8 +40,8 @@ def serve(port): try: s.bind(('', port)) s.listen(1) - except socket.error, e: - print "Cannot connect", e + except socket.error as e: + print("Cannot connect", e) sys.exit(EXIT_UNAVAILABLE) (conn, addr) = s.accept() @@ -69,7 +69,7 @@ def serve(port): return message[:-len(EOM)] def error(msg): - print "ERROR: %s" % (msg) + print("ERROR: %s" % (msg)) sys.exit(1) @@ -79,4 +79,4 @@ if len(sys.argv) < 2: port = int(sys.argv[1]) body = serve(port) -print body +print(body) From b2a01c15b075fded0fb2e46bcafc179dc91de751 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sun, 9 Jan 2022 15:01:38 +0100 Subject: [PATCH 16/27] Fix auto-migrated code - Use b'' (byte strings) where appropriate. - Fix indentation. - Replace python2.x references with python3.x. --- Makefile | 2 +- gpg-mailgate.py | 20 ++++++++++---------- test/e2e_test.py | 12 ++++++------ test/relay.py | 29 +++++++++++++++++------------ 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index a14332a..7184529 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON = python2.7 +PYTHON = python3.8 .PHONY: test pre-clean clean diff --git a/gpg-mailgate.py b/gpg-mailgate.py index e520aa5..39fcc61 100755 --- a/gpg-mailgate.py +++ b/gpg-mailgate.py @@ -255,7 +255,7 @@ def decrypt_inline_with_attachments( payloads, success, message = None ): # There was no encrypted payload found, so the original payload is attached message.attach(payload) - return message, success + return message, success def decrypt_inline_without_attachments( decrypted_message ): @@ -379,9 +379,9 @@ def gpg_encrypt( raw_message, recipients ): raw_message_mime['X-GPG-Mailgate'] = 'Encrypted by GPG Mailgate' if 'Content-Transfer-Encoding' in raw_message_mime: - raw_message_mime.replace_header('Content-Transfer-Encoding', '8BIT') - else: - raw_message_mime['Content-Transfer-Encoding'] = '8BIT' + raw_message_mime.replace_header('Content-Transfer-Encoding', '8BIT') + else: + raw_message_mime['Content-Transfer-Encoding'] = '8BIT' encrypted_payloads = encrypt_all_payloads_mime( raw_message_mime, gpg_to_cmdline_mime ) raw_message_mime.set_payload( encrypted_payloads ) @@ -396,9 +396,9 @@ def gpg_encrypt( raw_message, recipients ): raw_message_inline['X-GPG-Mailgate'] = 'Encrypted by GPG Mailgate' if 'Content-Transfer-Encoding' in raw_message_inline: - raw_message_inline.replace_header('Content-Transfer-Encoding', '8BIT') - else: - raw_message_inline['Content-Transfer-Encoding'] = '8BIT' + raw_message_inline.replace_header('Content-Transfer-Encoding', '8BIT') + else: + raw_message_inline['Content-Transfer-Encoding'] = '8BIT' encrypted_payloads = encrypt_all_payloads_inline( raw_message_inline, gpg_to_cmdline_inline ) raw_message_inline.set_payload( encrypted_payloads ) @@ -632,8 +632,8 @@ def send_msg( message, recipients ): log("Sending email to: <%s>" % '> <'.join( recipients )) relay = (cfg['relay']['host'], int(cfg['relay']['port'])) smtp = smtplib.SMTP(relay[0], relay[1]) - if 'relay' in cfg and 'starttls' in cfg['relay'] and cfg['relay']['starttls'] == 'yes': - smtp.starttls() + if 'relay' in cfg and 'starttls' in cfg['relay'] and cfg['relay']['starttls'] == 'yes': + smtp.starttls() smtp.sendmail( from_addr, recipients, message ) else: log("No recipient found") @@ -658,7 +658,7 @@ def sort_recipients( raw_message, from_addr, to_addrs ): return first_payload = first_payload.get_payload(decode=True) - if "-----BEGIN PGP MESSAGE-----" in first_payload and "-----END PGP MESSAGE-----" in first_payload: + if b"-----BEGIN PGP MESSAGE-----" in first_payload and b"-----END PGP MESSAGE-----" in first_payload: if verbose: log("Message is already encrypted as PGP/INLINE. Encryption aborted.") send_msg(raw_message.as_string(), recipients_left) diff --git a/test/e2e_test.py b/test/e2e_test.py index 0f7705f..619ce75 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python2 +#!/usr/local/bin/python3.8 # # gpg-mailgate @@ -32,7 +32,7 @@ from time import sleep RELAY_SCRIPT = "test/relay.py" CONFIG_FILE = "test/gpg-mailgate.conf" -PYTHON_BIN = "python2.7" +PYTHON_BIN = "python3.8" def build_config(config): cp = configparser.ConfigParser() @@ -55,7 +55,7 @@ def build_config(config): cp.set("enc_keymap", "alice@disposlab", "1CD245308F0963D038E88357973CF4D9387C44D7") cp.set("enc_keymap", "bob@disposlab", "19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67") - logging.debug("Created config with keyhome=%s, cert_path=%s and relay at port %d" % + logging.debug("Created config with keyhome=%s, cert_path=%s and relay at port %s" % (config["gpg_keyhome"], config["smime_certpath"], config["port"])) return cp @@ -128,14 +128,14 @@ logging.basicConfig(filename = config.get("tests", "e2e_log"), # Get raw values of log and date formats because they # contain %-sequences and we don't want them to be expanded # by the ConfigParser. - format = config.get("tests", "e2e_log_format", True), - datefmt = config.get("tests", "e2e_log_datefmt", True), + format = config.get("tests", "e2e_log_format", raw=True), + datefmt = config.get("tests", "e2e_log_datefmt", raw=True), level = logging.DEBUG) config_path = os.getcwd() + "/" + CONFIG_FILE write_test_config(config_path, - port = config.getint("relay", "port"), + port = config.get("relay", "port"), gpg_keyhome = config.get("dirs", "keys"), smime_certpath = config.get("dirs", "certs"), log_file = config.get("tests", "lacre_log")) diff --git a/test/relay.py b/test/relay.py index bc6969c..a8e3db7 100644 --- a/test/relay.py +++ b/test/relay.py @@ -14,38 +14,43 @@ import socket EXIT_UNAVAILABLE = 1 +ENCODING = 'utf-8' + BUFFER_SIZE = 4096 EOM = "\r\n.\r\n" LAST_LINE = -3 def welcome(msg): - return "220 %s\r\n" % (msg) + return b"220 %b\r\n" % (msg) -def ok(msg = "OK"): - return "250 %s\r\n" % (msg) +def ok(msg = b"OK"): + return b"250 %b\r\n" % (msg) def bye(): - return "251 Bye" + return b"251 Bye" def provide_message(): - return "354 Enter a message, ending it with a '.' on a line by itself\r\n" + return b"354 Enter a message, ending it with a '.' on a line by itself\r\n" def receive_and_confirm(session): session.recv(BUFFER_SIZE) session.sendall(ok()) +def localhost_at(port): + return ('127.0.0.1', port) + def serve(port): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: - s.bind(('', port)) + s.bind(localhost_at(port)) s.listen(1) except socket.error as e: print("Cannot connect", e) sys.exit(EXIT_UNAVAILABLE) (conn, addr) = s.accept() - conn.sendall(welcome("TEST SERVER")) + conn.sendall(welcome(b"TEST SERVER")) receive_and_confirm(conn) # Ignore HELO/EHLO receive_and_confirm(conn) # Ignore sender address @@ -57,8 +62,8 @@ def serve(port): # Consume until we get ., the end-of-message marker. message = '' while not message.endswith(EOM): - message += conn.recv(BUFFER_SIZE) - conn.sendall(ok("OK, id=test")) + message += conn.recv(BUFFER_SIZE).decode(ENCODING) + conn.sendall(ok(b"OK, id=test")) conn.recv(BUFFER_SIZE) conn.sendall(bye()) @@ -68,13 +73,13 @@ def serve(port): # Trim EOM marker as we're only interested in the message body. return message[:-len(EOM)] -def error(msg): +def error(msg, exit_code): print("ERROR: %s" % (msg)) - sys.exit(1) + sys.exit(exit_code) if len(sys.argv) < 2: - error("Usage: relay.py PORT_NUMBER") + error("Usage: relay.py PORT_NUMBER", EXIT_UNAVAILABLE) port = int(sys.argv[1]) body = serve(port) From 1e7d33c1df9c7810568bfab67d65edc1bd2c62fa Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sun, 9 Jan 2022 17:56:09 +0100 Subject: [PATCH 17/27] Handle bytes properly Fix bytes sequences handling after auto-migration. --- GnuPG/__init__.py | 3 ++- gpg-mailgate.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/GnuPG/__init__.py b/GnuPG/__init__.py index 6404720..b20c3d8 100644 --- a/GnuPG/__init__.py +++ b/GnuPG/__init__.py @@ -58,6 +58,7 @@ def public_keys( keyhome ): fingerprint = None email = None for line in p.stdout.readlines(): + line = line.decode('utf-8') if line[0:3] == LINE_FINGERPRINT: fingerprint = line.split(':')[POS_FINGERPRINT] if line[0:3] == LINE_USER_ID: @@ -120,7 +121,7 @@ def delete_key( keyhome, email ): class GPGEncryptor: def __init__(self, keyhome, recipients = None, charset = None): self._keyhome = keyhome - self._message = '' + self._message = b'' self._recipients = list() self._charset = charset if recipients != None: diff --git a/gpg-mailgate.py b/gpg-mailgate.py index 39fcc61..d360255 100755 --- a/gpg-mailgate.py +++ b/gpg-mailgate.py @@ -470,7 +470,7 @@ def encrypt_all_payloads_mime( message, gpg_to_cmdline ): def encrypt_payload( payload, gpg_to_cmdline, check_nested = True ): raw_payload = payload.get_payload(decode=True) - if check_nested and "-----BEGIN PGP MESSAGE-----" in raw_payload and "-----END PGP MESSAGE-----" in raw_payload: + if check_nested and b"-----BEGIN PGP MESSAGE-----" in raw_payload and b"-----END PGP MESSAGE-----" in raw_payload: if verbose: log("Message is already pgp encrypted. No nested encryption needed.") return payload @@ -596,9 +596,13 @@ def sanitize_case_sense( address ): if get_bool_from_cfg('default', 'mail_case_insensitive', 'yes'): address = address.lower() else: - splitted_address = address.split('@') + if isinstance(address, str): + sep = '@' + else: + sep = b'@' + splitted_address = address.split(sep) if len(splitted_address) > 1: - address = splitted_address[0] + '@' + splitted_address[1].lower() + address = splitted_address[0] + sep + splitted_address[1].lower() return address From 24f0c86d4f480feafed6a42206a10b95587a1ee0 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Mon, 10 Jan 2022 18:30:19 +0100 Subject: [PATCH 18/27] Tidy up tests - Makefile: add 'unittest' to .PHONY targets. - Remove unnecessary #! line from e2e_test.py. - Extract Python path to test/e2e.ini file. --- Makefile | 2 +- test/e2e.ini | 1 + test/e2e_test.py | 8 ++------ 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 7184529..4b31a51 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON = python3.8 -.PHONY: test pre-clean clean +.PHONY: test unittest pre-clean clean # # Run a set of end-to-end tests. diff --git a/test/e2e.ini b/test/e2e.ini index 2d6fb0b..c964ebf 100644 --- a/test/e2e.ini +++ b/test/e2e.ini @@ -31,6 +31,7 @@ certs: test/certs [tests] # Number of "test-*" sections in this file, describing test cases. cases: 3 +python_path: /usr/local/bin/python3.8 e2e_log: test/logs/e2e.log e2e_log_format: %(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s e2e_log_datefmt: %Y-%m-%d %H:%M:%S diff --git a/test/e2e_test.py b/test/e2e_test.py index 619ce75..d04216a 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -1,5 +1,3 @@ -#!/usr/local/bin/python3.8 - # # gpg-mailgate # @@ -32,8 +30,6 @@ from time import sleep RELAY_SCRIPT = "test/relay.py" CONFIG_FILE = "test/gpg-mailgate.conf" -PYTHON_BIN = "python3.8" - def build_config(config): cp = configparser.ConfigParser() @@ -95,10 +91,10 @@ def execute_e2e_test(case_name, config, config_path): test_command = "GPG_MAILGATE_CONFIG=%s %s gpg-mailgate.py %s < %s" % ( config_path, - PYTHON_BIN, + config.get("tests", "python_path"), config.get(case_name, "to"), config.get(case_name, "in")) - result_command = "%s %s %d" % (PYTHON_BIN, config.get("relay", "script"), config.getint("relay", "port")) + result_command = "%s %s %d" % (config.get("tests", "python_path"), config.get("relay", "script"), config.getint("relay", "port")) logging.debug("Spawning relay: '%s'" % (result_command)) pipe = os.popen(result_command, 'r') From 435528de433b410caccbd9d4662d63f65b74db8f Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Mon, 10 Jan 2022 19:32:46 +0100 Subject: [PATCH 19/27] Add an E2E test case with an already encrypted message --- test/e2e.ini | 8 +++++++- test/msgin/ed2ed.msg | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 test/msgin/ed2ed.msg diff --git a/test/e2e.ini b/test/e2e.ini index c964ebf..feac655 100644 --- a/test/e2e.ini +++ b/test/e2e.ini @@ -30,7 +30,7 @@ certs: test/certs [tests] # Number of "test-*" sections in this file, describing test cases. -cases: 3 +cases: 4 python_path: /usr/local/bin/python3.8 e2e_log: test/logs/e2e.log e2e_log_format: %(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s @@ -54,3 +54,9 @@ descr: Clear text message to a user with an Ed25519 key to: bob@disposlab in: test/msgin/clear2ed.msg out: -----BEGIN PGP MESSAGE----- + +[case-4] +descr: Encrypted message to a user with an Ed25519 key +to: bob@disposlab +in: test/msgin/ed2ed.msg +out: -----BEGIN PGP MESSAGE----- diff --git a/test/msgin/ed2ed.msg b/test/msgin/ed2ed.msg new file mode 100644 index 0000000..90b73a8 --- /dev/null +++ b/test/msgin/ed2ed.msg @@ -0,0 +1,13 @@ +From: Dave +To: Bob +Subject: Test +Content-Transfer-Encoding: 7bit + +-----BEGIN PGP MESSAGE----- + +hF4DujWCoRS24dYSAQdAyGDF9Us11JDr8+XPmvlJHsMS7A4UBIcCiresJyZpSxYw +Cqcugy5AX5fgSAiL1Cd2b1zpQ/rYdTWkFYMVbH4jBEoPC3z/aSd+hTnneJFDUdXl +0koBDIw7NQylu6SrW+Y/DmXgalIHtwACuKivJTq/z9jdwFScV7adRR/VO53Inah3 +L1+Ho7Zta95AYW3UPu71Gw3rrkfjY4uGDiFAFg== +=yTzD +-----END PGP MESSAGE----- From 5a8d2c0108c0c9c3cb5f8e91fa115087c4d6bfe8 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Mon, 10 Jan 2022 19:48:44 +0100 Subject: [PATCH 20/27] Add E2E cases: signed cleartext and multipart/encrypted --- test/e2e.ini | 14 +++++++++++- test/msgin/multipart2rsa.msg | 43 ++++++++++++++++++++++++++++++++++++ test/msgin/signed.msg | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 test/msgin/multipart2rsa.msg create mode 100644 test/msgin/signed.msg diff --git a/test/e2e.ini b/test/e2e.ini index feac655..8ad8577 100644 --- a/test/e2e.ini +++ b/test/e2e.ini @@ -30,7 +30,7 @@ certs: test/certs [tests] # Number of "test-*" sections in this file, describing test cases. -cases: 4 +cases: 6 python_path: /usr/local/bin/python3.8 e2e_log: test/logs/e2e.log e2e_log_format: %(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s @@ -60,3 +60,15 @@ descr: Encrypted message to a user with an Ed25519 key to: bob@disposlab in: test/msgin/ed2ed.msg out: -----BEGIN PGP MESSAGE----- + +[case-5] +descr: Signed message to a user with an Ed25519 key +to: bob@disposlab +in: test/msgin/signed.msg +out: -----BEGIN PGP MESSAGE----- + +[case-6] +descr: Multipart encrypted message to a user with an Ed25519 key. +to: bob@disposlab +in: test/msgin/multipart2rsa.msg +out: -----BEGIN PGP MESSAGE----- diff --git a/test/msgin/multipart2rsa.msg b/test/msgin/multipart2rsa.msg new file mode 100644 index 0000000..3b8f623 --- /dev/null +++ b/test/msgin/multipart2rsa.msg @@ -0,0 +1,43 @@ +Date: Sun, 18 Jul 2021 16:53:45 +0200 +From: User Alice +To: User Bob +Subject: encrypted +Message-ID: +MIME-Version: 1.0 +Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; + boundary="95hZs/zeBetwhuEy" +Content-Disposition: inline +Status: RO +Content-Length: 1140 +Lines: 30 + + +--95hZs/zeBetwhuEy +Content-Type: application/pgp-encrypted +Content-Disposition: attachment + +Version: 1 + +--95hZs/zeBetwhuEy +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="msg.asc" + +-----BEGIN PGP MESSAGE----- + +hQGMA/vsqjpkmZurAQwAnb+2kDPgVFWVLkafuzVJGqFWKNtdVsvk7I1zhzFw5Hsr +h4irSHcH0X0QjaHprNiMBDfIZaCx5VVsvGYLiu/iQkdVPXItugTpln8aAvDt8/Bp +Hse69tgG5S9o4fPK4K2bMjNdomclDdz51cu9NXYjk/6OtzVwcSypyEmxgw24Oo1+ +Q8KfZN9n6VTXGNlrV9KnAZYs/5aaSABTeC+cDvOcjDbPAmwDHYS3qsbITYoGHnEz +QfPIakYWPtPWkajhm4Z/iyEUSTeqew1/gAJ8sZnJpV0eg1Cr/44XgklZKFr8aJgk +SG8PkQxsyzAZklpwMSWdbb+t9a5nEKvky3zMpdmS1GE7ubTO7nQ1geUdBiv1UUNh +BY9d4nlGirqxX1MZUTGZidJgCy0365xbJSKkU0yFFW2uWtCKzJTEQBk3YZkNmnGH +h8BiVvMhQ8SxKBRPeH6Zb6HHlbcgkPvJAAI4VLqkZPCBvp9irmcdFGmrgCWLxzgk +sIjYGLA+ZuSXOKuAssXE0sAbASPAkUJRTIjzXFrCnr/MB3ZonESH01fsbsX+E/Qi ++2oLrgjjPHcPq76fvdO6fJP6c1pM8TlOoZKn/RkPm1llULtOn4n5JZJjeUA0F2ID +Te/U9i4YtcFZbuvw2bjeu8sAf77U6O3iTTBWkPWQT3H4YMskQc7lS1Mug6A9HL/n +TQvAwh2MIveYyEy/y/dKeFUbpSKxyOInhTg1XtYFiT8bzEF7OEJLU9GyF5oMs67d +o12uYlEnPhWz9oZp11aSdnyeADpVu6BQsPbwfTifcpajQSarH5sG8+rDSPju +=7CnH +-----END PGP MESSAGE----- + +--95hZs/zeBetwhuEy-- diff --git a/test/msgin/signed.msg b/test/msgin/signed.msg new file mode 100644 index 0000000..917291e --- /dev/null +++ b/test/msgin/signed.msg @@ -0,0 +1,39 @@ +Date: Sun, 18 Jul 2021 12:08:41 +0200 +From: User Alice +To: User Bob +Subject: signed +Message-ID: +MIME-Version: 1.0 +Content-Type: multipart/signed; micalg=pgp-sha256; + protocol="application/pgp-signature"; boundary="U/XjR71RAixRcb28" +Content-Disposition: inline +Status: RO +Content-Length: 870 +Lines: 26 + + +--U/XjR71RAixRcb28 +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline + +A signed msg. + +--U/XjR71RAixRcb28 +Content-Type: application/pgp-signature; name="signature.asc" + +-----BEGIN PGP SIGNATURE----- + +iQGzBAEBCAAdFiEEHNJFMI8JY9A46INXlzz02Th8RNcFAmDz/aUACgkQlzz02Th8 +RNdtOQv/ca8c51KoVq7CyPJUr54n4DEk/LlYniR0W51tL2a4rQxyF2AxqjdI8T4u +bT1+bqPNYgegesyCLokeZKqhLVtCH+UVOTdtUq5bB1J7ALuuVTOIdR5woMBBsazV +ETYEMzL6y2sGPW92ynriEw6B9pPnFKFPhOOZLrnMzM8CpkTfNmGoej+EdV74s0z4 +RayKu/WaZ1Dtx2Vy2YDtG36p/Y3n62bnzQJCRyPYfrmCxH5X5i5oibQwxLROCFNE +4X3iVZLPHFg/DS9m4L7mBe0MJewGa1oPFr7t3ZfJ+24aJ/AvUv5uQIO+s6a7AcjD +Pgw/IjeM/uZdPrzniZI2zsWEgsjRCL1fj49XWVNkTHrWCqLvkBg+suucNO2SR0/d +ps+RP5mkJJHaSZyPpxwo9/PHKX67Mkpn/uEXlE8nV6IqKoXRzr1N0qwyhvbZQZLD +FMumxx/eOSiOpaiRhGhoZiUpf+VdnV/1ClpAcdbthy/psx/CMYVblAM8xg74NR9+ +Q/WlFbRl +=uMdE +-----END PGP SIGNATURE----- + +--U/XjR71RAixRcb28-- From c81c6e6e0d991219b4d0729b3377d1cedef85ad1 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Mon, 10 Jan 2022 20:22:17 +0100 Subject: [PATCH 21/27] Remove hardcoded python3.8 path - Let the user overwrite Python binary name while calling make. - Use environment variable set by make to instruct e2e_test.py which binary it should call to execute Python code. --- Makefile | 14 ++++++++++++-- test/e2e.ini | 1 - test/e2e_test.py | 6 ++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 4b31a51..9c4ce7b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,17 @@ -PYTHON = python3.8 - +.POSIX: .PHONY: test unittest pre-clean clean +# +# On systems where Python 3.x binary has a different name, just +# overwrite the name/path on the command line, like: +# +# make test PYTHON=/usr/local/bin/python3.8 +# +# This marco is passed via environment to test/e2e_test.py, where it's +# used to compute further commands. +# +PYTHON = python3 + # # Run a set of end-to-end tests. # diff --git a/test/e2e.ini b/test/e2e.ini index 8ad8577..134a005 100644 --- a/test/e2e.ini +++ b/test/e2e.ini @@ -31,7 +31,6 @@ certs: test/certs [tests] # Number of "test-*" sections in this file, describing test cases. cases: 6 -python_path: /usr/local/bin/python3.8 e2e_log: test/logs/e2e.log e2e_log_format: %(asctime)s %(pathname)s:%(lineno)d %(levelname)s [%(funcName)s] %(message)s e2e_log_datefmt: %Y-%m-%d %H:%M:%S diff --git a/test/e2e_test.py b/test/e2e_test.py index d04216a..a3617d6 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -88,13 +88,15 @@ def execute_e2e_test(case_name, config, config_path): config file. Each of these sections should contain following properties: 'descr', 'to', 'in' and 'out'. """ + # This environment variable is set in Makefile. + python_path = os.getenv('PYTHON', 'python3') test_command = "GPG_MAILGATE_CONFIG=%s %s gpg-mailgate.py %s < %s" % ( config_path, - config.get("tests", "python_path"), + python_path, config.get(case_name, "to"), config.get(case_name, "in")) - result_command = "%s %s %d" % (config.get("tests", "python_path"), config.get("relay", "script"), config.getint("relay", "port")) + result_command = "%s %s %d" % (python_path, config.get("relay", "script"), config.getint("relay", "port")) logging.debug("Spawning relay: '%s'" % (result_command)) pipe = os.popen(result_command, 'r') From a201265f871c45eb5ee44b90fea1cf4d2ce81e39 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Wed, 19 Jan 2022 21:57:46 +0100 Subject: [PATCH 22/27] Rework how E2E tests are executed Tests kept breaking (not failing) randomly with "Broken pipe" errors. Rework how processes are spawned to make sure that it doesn't break them again. --- test/e2e_test.py | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/test/e2e_test.py b/test/e2e_test.py index a3617d6..efc9e3c 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -20,6 +20,8 @@ import os import sys +import subprocess + import difflib import configparser @@ -70,7 +72,7 @@ def load_file(name): contents = f.read() f.close() - return contents + return bytes(contents, 'utf-8') def report_result(message_file, expected, test_output): status = None @@ -91,23 +93,35 @@ def execute_e2e_test(case_name, config, config_path): # This environment variable is set in Makefile. python_path = os.getenv('PYTHON', 'python3') - test_command = "GPG_MAILGATE_CONFIG=%s %s gpg-mailgate.py %s < %s" % ( - config_path, - python_path, - config.get(case_name, "to"), - config.get(case_name, "in")) - result_command = "%s %s %d" % (python_path, config.get("relay", "script"), config.getint("relay", "port")) + gpglacre_cmd = [python_path, + "gpg-mailgate.py", + config.get(case_name, "to")] - logging.debug("Spawning relay: '%s'" % (result_command)) - pipe = os.popen(result_command, 'r') + relay_cmd = [python_path, + config.get("relay", "script"), + config.get("relay", "port")] - logging.debug("Spawning GPG-Lacre: '%s'" % (test_command)) - msgin = os.popen(test_command, 'w') - msgin.write(load_file(config.get(case_name, "in"))) - msgin.close() + logging.debug("Spawning relay: '%s'" % (relay_cmd)) + relay_proc = subprocess.Popen(relay_cmd, + stdin = None, + stdout = subprocess.PIPE) - testout = pipe.read() - pipe.close() + logging.debug("Spawning GPG-Lacre: '%s', stdin = %s" + % (gpglacre_cmd, + config.get(case_name, "in"))) + + # pass PATH because otherwise it would be dropped + gpglacre_proc = subprocess.run(gpglacre_cmd, + input = load_file(config.get(case_name, "in")), + capture_output = True, + env = {"GPG_MAILGATE_CONFIG": config_path, + "PATH": os.getenv("PATH")}) + + # Let the relay process the data. + relay_proc.wait() + + (testout, _) = relay_proc.communicate() + testout = testout.decode('utf-8') logging.debug("Read %d characters of test output: '%s'" % (len(testout), testout)) From 03fc3d138e277b432ecc1488a5bec80a54f20aa2 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Wed, 19 Jan 2022 22:16:27 +0100 Subject: [PATCH 23/27] Update testing documentation - Explain how to specify Python binary path used during tests. - Mention test logs. --- doc/testing.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/testing.md b/doc/testing.md index a462b0f..d018770 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -11,6 +11,10 @@ To run tests, use command `make test` or `make unittest`. Tests produce some helpful logs, so inspect contents of `test/logs` directory if something goes wrong. +If your system's Python binary isn't found in your `$PATH` or you want to use +a specific binary, use make's macro overriding: `make test +PYTHON=/path/to/python`. + ## Key building blocks - *Test Script* (`test/e2e_test.py`) that orchestrates the other components. @@ -29,3 +33,8 @@ Currently tests only check if the message has been encrypted, without verifying that the correct key has been used. That's because we don't know (yet) how to have a reproducible encrypted message. Option `--faked-system-time` wasn't enough to produce identical output. + +## Troubleshooting + +When things go wrong, be sure to study `test/logs/e2e.log` and +`test/logs/gpg-mailgate.log` files -- they contain some useful information. From 67a938c049dd15215031f02d2075640a3e4971e7 Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Tue, 25 Jan 2022 20:32:17 +0100 Subject: [PATCH 24/27] GnuPG.add_key: Use build_command --- GnuPG/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GnuPG/__init__.py b/GnuPG/__init__.py index b20c3d8..8bcfebd 100644 --- a/GnuPG/__init__.py +++ b/GnuPG/__init__.py @@ -102,7 +102,7 @@ def confirm_key( content, email ): # adds a key and ensures it has the given email address def add_key( keyhome, content ): - p = subprocess.Popen( ['/usr/bin/gpg', '--homedir', keyhome, '--import', '--batch'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + p = subprocess.Popen( build_command(keyhome, '--import', '--batch'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) p.communicate(input=content) p.wait() From 9e17726e39f0407f32926b866bbaaaee021bb8cb Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sun, 6 Feb 2022 00:13:15 +0100 Subject: [PATCH 25/27] Use f-strings for formatting --- test/e2e_test.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/e2e_test.py b/test/e2e_test.py index efc9e3c..f937fa9 100644 --- a/test/e2e_test.py +++ b/test/e2e_test.py @@ -53,19 +53,18 @@ def build_config(config): cp.set("enc_keymap", "alice@disposlab", "1CD245308F0963D038E88357973CF4D9387C44D7") cp.set("enc_keymap", "bob@disposlab", "19CF4B47ECC9C47AFA84D4BD96F39FDA0E31BB67") - logging.debug("Created config with keyhome=%s, cert_path=%s and relay at port %s" % - (config["gpg_keyhome"], config["smime_certpath"], config["port"])) + logging.debug(f"Created config with keyhome={config['gpg_keyhome']}, cert_path={config['smime_certpath']} and relay at port {config['port']}") return cp def write_test_config(outfile, **config): - logging.debug("Generating configuration with %s" % repr(config)) + logging.debug(f"Generating configuration with {config!r}") out = open(outfile, "w+") cp = build_config(config) cp.write(out) out.close() - logging.debug("Wrote configuration to %s" % outfile) + logging.debug(f"Wrote configuration to {outfile}") def load_file(name): f = open(name, 'r') @@ -101,14 +100,12 @@ def execute_e2e_test(case_name, config, config_path): config.get("relay", "script"), config.get("relay", "port")] - logging.debug("Spawning relay: '%s'" % (relay_cmd)) + logging.debug(f"Spawning relay: {relay_cmd}") relay_proc = subprocess.Popen(relay_cmd, stdin = None, stdout = subprocess.PIPE) - logging.debug("Spawning GPG-Lacre: '%s', stdin = %s" - % (gpglacre_cmd, - config.get(case_name, "in"))) + logging.debug(f"Spawning GPG-Lacre: {gpglacre_cmd}, stdin = {config.get(case_name, 'in')}") # pass PATH because otherwise it would be dropped gpglacre_proc = subprocess.run(gpglacre_cmd, @@ -123,7 +120,7 @@ def execute_e2e_test(case_name, config, config_path): (testout, _) = relay_proc.communicate() testout = testout.decode('utf-8') - logging.debug("Read %d characters of test output: '%s'" % (len(testout), testout)) + logging.debug(f"Read {len(testout)} characters of test output: '{testout}'") report_result(config.get(case_name, "in"), config.get(case_name, "out"), testout) @@ -153,8 +150,8 @@ write_test_config(config_path, log_file = config.get("tests", "lacre_log")) for case_no in range(1, config.getint("tests", "cases")+1): - case_name = "case-%d" % (case_no) - logging.info("Executing %s: %s", case_name, config.get(case_name, "descr")) + case_name = f"case-{case_no}" + logging.info(f"Executing {case_name}: {config.get(case_name, 'descr')}") execute_e2e_test(case_name, config, config_path) From c4927d2722580e6c329ec755abcd31d3ecdd77aa Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Sun, 6 Feb 2022 00:27:38 +0100 Subject: [PATCH 26/27] Avoid unnecessary list creation --- gpg-mailgate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gpg-mailgate.py b/gpg-mailgate.py index d360255..5c8090a 100755 --- a/gpg-mailgate.py +++ b/gpg-mailgate.py @@ -94,7 +94,7 @@ def gpg_decrypt( raw_message, recipients ): keys[fingerprint] = sanitize_case_sense(keys[fingerprint]) for to in recipients: - if to in list(keys.values()) and not get_bool_from_cfg('default', 'dec_keymap_only', 'yes'): + if to in keys.values() and not get_bool_from_cfg('default', 'dec_keymap_only', 'yes'): gpg_to.append(to) # Is this recipient defined in regex for default decryption? elif not (dec_regex is None) and not (re.match(dec_regex, to) is None): @@ -106,7 +106,7 @@ def gpg_decrypt( raw_message, recipients ): if not cfg['dec_keymap'][to] in keys: log("Key '%s' in decryption keymap not found in keyring for email address '%s'. Won't decrypt." % (cfg['dec_keymap'][to], to)) # Avoid unwanted encryption if set - if to in list(keys.values()) and get_bool_from_cfg('default', 'failsave_dec', 'yes'): + if to in keys.values() and get_bool_from_cfg('default', 'failsave_dec', 'yes'): noenc_to.append(to) else: ungpg_to.append(to) @@ -116,7 +116,7 @@ def gpg_decrypt( raw_message, recipients ): if verbose: log("Recipient (%s) not in PGP domain list for decrypting." % to) # Avoid unwanted encryption if set - if to in list(keys.values()) and get_bool_from_cfg('default', 'failsave_dec', 'yes'): + if to in keys.values() and get_bool_from_cfg('default', 'failsave_dec', 'yes'): noenc_to.append(to) else: ungpg_to.append(to) @@ -317,7 +317,7 @@ def gpg_encrypt( raw_message, recipients ): log("Key '%s' in encrypt keymap not found in keyring for email address '%s'." % (cfg['enc_keymap'][to], to)) # Check if key in keychain is present - if to in list(keys.values()) and not get_bool_from_cfg('default', 'enc_keymap_only', 'yes'): + if to in keys.values() and not get_bool_from_cfg('default', 'enc_keymap_only', 'yes'): gpg_to.append( (to, to) ) continue @@ -341,7 +341,7 @@ def gpg_encrypt( raw_message, recipients ): ungpg_to.append(to) if gpg_to != list(): - log("Encrypting email to: %s" % ' '.join( [x[0] for x in gpg_to] )) + log("Encrypting email to: %s" % ' '.join( x[0] for x in gpg_to )) # Getting PGP style for recipient gpg_to_smtp_mime = list() From 59b932abaa13d25c505af22f02fa520ce0a35a3b Mon Sep 17 00:00:00 2001 From: "Piotr F. Mieszkowski" Date: Tue, 8 Mar 2022 21:59:20 +0100 Subject: [PATCH 27/27] Remove support for decrypting emails --- gpg-mailgate.py | 228 +----------------------------------------------- 1 file changed, 1 insertion(+), 227 deletions(-) diff --git a/gpg-mailgate.py b/gpg-mailgate.py index 5c8090a..02e1c80 100755 --- a/gpg-mailgate.py +++ b/gpg-mailgate.py @@ -70,227 +70,6 @@ raw_message = email.message_from_string( raw ) from_addr = raw_message['From'] to_addrs = sys.argv[1:] -def gpg_decrypt( raw_message, recipients ): - - gpg_to = list() - ungpg_to = list() - - # This is needed to avoid encryption if decryption is set to keymap only, - # private key is present but not in keymap. - noenc_to = list() - - if not get_bool_from_cfg('gpg', 'keyhome'): - log("No valid entry for gpg keyhome. Decryption aborted.") - return recipients - - keys = GnuPG.private_keys( cfg['gpg']['keyhome'] ) - - if get_bool_from_cfg('default', 'dec_regex'): - dec_regex = cfg['default']['dec_regex'] - else: - dec_regex = None - - for fingerprint in keys: - keys[fingerprint] = sanitize_case_sense(keys[fingerprint]) - - for to in recipients: - if to in keys.values() and not get_bool_from_cfg('default', 'dec_keymap_only', 'yes'): - gpg_to.append(to) - # Is this recipient defined in regex for default decryption? - elif not (dec_regex is None) and not (re.match(dec_regex, to) is None): - log("Using default decrytion defined in dec_regex for recipient '%s'" % to) - gpg_to.append(to) - elif get_bool_from_cfg('dec_keymap', to): - log("Decrypt keymap has key '%s'" % cfg['dec_keymap'][to] ) - # Check we've got a matching key! If not, decline to attempt decryption. The key is checked for safty reasons. - if not cfg['dec_keymap'][to] in keys: - log("Key '%s' in decryption keymap not found in keyring for email address '%s'. Won't decrypt." % (cfg['dec_keymap'][to], to)) - # Avoid unwanted encryption if set - if to in keys.values() and get_bool_from_cfg('default', 'failsave_dec', 'yes'): - noenc_to.append(to) - else: - ungpg_to.append(to) - else: - gpg_to.append(to) - else: - if verbose: - log("Recipient (%s) not in PGP domain list for decrypting." % to) - # Avoid unwanted encryption if set - if to in keys.values() and get_bool_from_cfg('default', 'failsave_dec', 'yes'): - noenc_to.append(to) - else: - ungpg_to.append(to) - - if gpg_to != list(): - send_msg( gpg_decrypt_all_payloads( raw_message ).as_string(), gpg_to ) - - if noenc_to != list(): - log("Do not try to encrypt mails for: %s" % ', '.join( noenc_to )) - send_msg(raw_message.as_string(), noenc_to) - - return ungpg_to - -def gpg_decrypt_all_payloads( message ): - - # We don't want to modify the original message - decrypted_message = copy.deepcopy(message) - - # Check if message is PGP/MIME encrypted - if not (message.get_param('protocol') is None) and message.get_param('protocol') == 'application/pgp-encrypted' and message.is_multipart(): - decrypted_message = decrypt_mime(decrypted_message) - - # At this point the message could only be PGP/INLINE encrypted, unencrypted or - # encrypted with a mechanism not covered by GPG-Mailgate - elif get_bool_from_cfg('default', 'no_inline_dec', 'no'): - # Check if message is PGP/INLINE encrypted and has attachments (or unencrypted with attachments) - if message.is_multipart(): - - # Set message's payload to list so payloads can be attached later on - decrypted_message.set_payload(list()) - - # We only need to hand over the original message here. Not needed for other decrypt implementations. - decrypted_message, success = decrypt_inline_with_attachments(message, False, decrypted_message) - - # Add header here to avoid it being appended several times - if get_bool_from_cfg('default', 'add_header', 'yes') and success: - decrypted_message['X-GPG-Mailgate'] = 'Decrypted by GPG Mailgate' - - # Check if message is PGP/INLINE encrypted without attachments (or unencrypted without attachments) - else: - decrypted_message = decrypt_inline_without_attachments(decrypted_message) - - return decrypted_message - -def decrypt_mime( decrypted_message ): - # Please note: Signatures will disappear while decrypting and will not be checked - - # Getting the part which should be PGP encrypted (according to RFC) - msg_content = decrypted_message.get_payload(1).get_payload() - - if "-----BEGIN PGP MESSAGE-----" in msg_content and "-----END PGP MESSAGE-----" in msg_content: - start = msg_content.find("-----BEGIN PGP MESSAGE-----") - end = msg_content.find("-----END PGP MESSAGE-----") - decrypted_payload, decrypt_success = decrypt_payload(msg_content[start:end + 25]) - - if decrypt_success: - # Making decrypted_message a "normal" unencrypted message - decrypted_message.del_param('protocol') - decrypted_message.set_type(decrypted_payload.get_content_type()) - - # Restore Content-Disposition header from original message - if not (decrypted_payload.get('Content-Disposition') is None): - if not (decrypted_message.get('Content-Disposition') is None): - decrypted_message.replace_header('Content-Disposition', decrypted_payload.get('Content-Disposition')) - else: - decrypted_message.set_param(decrypted_payload.get('Content-Disposition'), "", 'Content-Disposition') - - if decrypted_payload.is_multipart(): - # Clear message's original payload and insert the decrypted payloads - decrypted_message.set_payload(list()) - decrypted_message = generate_message_from_payloads( decrypted_payload, decrypted_message ) - decrypted_message.preamble = "This is a multi-part message in MIME format" - else: - decrypted_message.set_payload(decrypted_payload.get_payload()) - decrypted_message.preamble = None - - - if get_bool_from_cfg('default', 'add_header', 'yes'): - decrypted_message['X-GPG-Mailgate'] = 'Decrypted by GPG Mailgate' - - # If decryption fails, decrypted_message is equal to the original message - return decrypted_message - -def decrypt_inline_with_attachments( payloads, success, message = None ): - - if message is None: - message = email.mime.multipart.MIMEMultipart(payloads.get_content_subtype()) - - for payload in payloads.get_payload(): - if( isinstance(payload.get_payload(), list) ): - # Take care of cascaded MIME messages - submessage, subsuccess = decrypt_inline_with_attachments( payload, success ) - message.attach(submessage) - success = success or subsuccess - else: - msg_content = payload.get_payload() - - # Getting values for different implementations as PGP/INLINE is not implemented - # the same on different clients - pgp_inline_tags = "-----BEGIN PGP MESSAGE-----" in msg_content and "-----END PGP MESSAGE-----" in msg_content - attachment_filename = payload.get_filename() - - if pgp_inline_tags or not (attachment_filename is None) and not (re.search('.\.pgp$', attachment_filename) is None): - if pgp_inline_tags: - start = msg_content.find("-----BEGIN PGP MESSAGE-----") - end = msg_content.find("-----END PGP MESSAGE-----") - decrypted_payload, decrypt_success = decrypt_payload(msg_content[start:end + 25]) - # Some implementations like Enigmail have strange interpretations of PGP/INLINE - # This tries to cope with it as good as possible. - else: - build_message = """ ------BEGIN PGP MESSAGE----- - -%s ------END PGP MESSAGE-----""" % msg_content - - decrypted_payload, decrypt_success = decrypt_payload(build_message) - - # Was at least one decryption successful? - success = success or decrypt_success - - if decrypt_success: - - if not (attachment_filename is None): - attachment_filename = re.sub('\.pgp$', '', attachment_filename) - payload.set_param('filename', attachment_filename, 'Content-Disposition') - payload.set_param('name', attachment_filename, 'Content-Type') - # Need this nasty hack to avoid double blank lines at beginning of message - payload.set_payload(decrypted_payload.as_string()[1:]) - - message.attach(payload) - else: - # Message could not be decrypted, so non-decrypted message is attached - message.attach(payload) - else: - # There was no encrypted payload found, so the original payload is attached - message.attach(payload) - - return message, success - -def decrypt_inline_without_attachments( decrypted_message ): - - msg_content = decrypted_message.get_payload() - if "-----BEGIN PGP MESSAGE-----" in msg_content and "-----END PGP MESSAGE-----" in msg_content: - start = msg_content.find("-----BEGIN PGP MESSAGE-----") - end = msg_content.find("-----END PGP MESSAGE-----") - decrypted_payload, decrypt_success = decrypt_payload(msg_content[start:end + 25]) - - if decrypt_success: - # Need this nasty hack to avoid double blank lines at beginning of message - decrypted_message.set_payload(decrypted_payload.as_string()[1:]) - - if get_bool_from_cfg('default', 'add_header', 'yes'): - decrypted_message['X-GPG-Mailgate'] = 'Decrypted by GPG Mailgate' - - # If message was not encrypted, this will just return the original message - return decrypted_message - -def decrypt_payload( payload ): - - gpg = GnuPG.GPGDecryptor( cfg['gpg']['keyhome'] ) - gpg.update( payload ) - decrypted_data, returncode = gpg.decrypt() - if verbose: - log("Return code from decryption=%d (0 indicates success)." % returncode) - if returncode != 0: - log("Decrytion failed with return code %d. Decryption aborted." % returncode) - return payload, False - - # Decryption always generate a new message - decrypted_msg = email.message_from_string(decrypted_data) - - return decrypted_msg, True - def gpg_encrypt( raw_message, recipients ): if not get_bool_from_cfg('gpg', 'keyhome'): @@ -459,7 +238,7 @@ def encrypt_all_payloads_mime( message, gpg_to_cmdline ): junk_str = junk_msg.as_string() # WTF! Without this, get_boundary() will return 'None'! boundary = junk_msg.get_boundary() - # This also modifies the boundary in the body of the message, ie it gets parsed. + # This also modifies the boundary in the body of the message, ie it gets parsed. if 'Content-Type' in message: message.replace_header('Content-Type', "multipart/encrypted; protocol=\"application/pgp-encrypted\";\nboundary=\"%s\"\n" % boundary) else: @@ -648,11 +427,6 @@ def sort_recipients( raw_message, from_addr, to_addrs ): for recipient in to_addrs: recipients_left.append(sanitize_case_sense(recipient)) - # Decrypt mails for recipients with known private PGP keys - recipients_left = gpg_decrypt(raw_message, recipients_left) - if recipients_left == list(): - return - # There is no need for nested encryption first_payload = get_first_payload(raw_message) if first_payload.get_content_type() == 'application/pkcs7-mime':