From 9b2772b171dbe3b422cca924ca45d80b353bd49d Mon Sep 17 00:00:00 2001 From: Ivanq Date: Tue, 13 Oct 2020 19:12:46 +0300 Subject: [PATCH 1/3] Use more unique yet short paths for logging --- src/Debug/Debug.py | 72 +++++++++++++++++++++++++++++++------------ src/Test/TestDebug.py | 52 +++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 src/Test/TestDebug.py diff --git a/src/Debug/Debug.py b/src/Debug/Debug.py index 12f084bc..dec07919 100644 --- a/src/Debug/Debug.py +++ b/src/Debug/Debug.py @@ -28,7 +28,12 @@ def formatExceptionMessage(err): return "%s: %s" % (err_type, err_message) -python_lib_dir = os.path.dirname(os.__file__) +python_lib_dirs = [path for path in sys.path if path.endswith("/site-packages") or path.endswith("/dist-packages")] +python_lib_dirs.append(os.path.dirname(os.__file__)) # TODO: check if returns the correct path for PyPy + +root_dir = os.path.realpath(__file__) +for _ in range(3): + root_dir = os.path.dirname(root_dir) def formatTraceback(items, limit=None, fold_builtin=True): @@ -40,30 +45,59 @@ def formatTraceback(items, limit=None, fold_builtin=True): for path, line in items: i += 1 is_last = i == len(items) - dir_name, file_name = os.path.split(path.replace("\\", "/")) + path = path.replace("\\", "/") - plugin_match = re.match(".*/plugins/(.+)$", dir_name) - if plugin_match: - file_title = "%s/%s" % (plugin_match.group(1), file_name) - is_prev_builtin = False - elif path.startswith(python_lib_dir): - if is_prev_builtin and not is_last and fold_builtin: - if back[-1] != "...": - back.append("...") - continue + if path in ("", ""): + file_title = "(importlib)" + is_builtin = True + is_skippable_builtin = True + else: + is_skippable_builtin = False + for base in python_lib_dirs: + if path.startswith(base + "/"): + file_title = path[len(base + "/"):] + module_name, *tail = file_title.split("/") + if module_name.endswith(".py"): + module_name = module_name[:-3] + file_title = "/".join(["<%s>" % module_name] + tail) + is_builtin = True + break else: - file_title = path.replace(python_lib_dir, "").replace("\\", "/").strip("/").replace("site-packages/", "") - is_prev_builtin = True - else: - file_title = file_name - is_prev_builtin = False + is_builtin = False + for base in (root_dir + "/src", root_dir + "/plugins", root_dir): + if path.startswith(base + "/"): + file_title = path[len(base + "/"):] + break + else: + # For unknown paths, do our best to hide absolute path + file_title = path + for needle in ("/zeronet/", "/core/"): + if needle in file_title.lower(): + file_title = "?/" + file_title[file_title.lower().rindex(needle) + len(needle):] - if file_title == prev_file_title: - back.append("%s" % line) + # Path compression: A/AB/ABC/X/Y.py -> ABC/X/Y.py + # E.g.: in 'Db/DbCursor.py' the directory part is unnecessary + if not file_title.startswith("/"): + prev_part = "" + for i, part in enumerate(file_title.split("/") + [""]): + if not part.startswith(prev_part): + break + prev_part = part + file_title = "/".join(file_title.split("/")[i - 1:]) + + if is_skippable_builtin and fold_builtin: + pass + elif is_builtin and is_prev_builtin and not is_last and fold_builtin: + if back[-1] != "...": + back.append("...") else: - back.append("%s line %s" % (file_title, line)) + if file_title == prev_file_title: + back.append("%s" % line) + else: + back.append("%s line %s" % (file_title, line)) prev_file_title = file_title + is_prev_builtin = is_builtin if limit and i >= limit: back.append("...") diff --git a/src/Test/TestDebug.py b/src/Test/TestDebug.py new file mode 100644 index 00000000..695a81a9 --- /dev/null +++ b/src/Test/TestDebug.py @@ -0,0 +1,52 @@ +from Debug import Debug +import gevent +import os + +import pytest + + +class TestDebug: + @pytest.mark.parametrize("items,expected", [ + (["@/src/A/B/C.py:17"], ["A/B/C.py line 17"]), # basic test + (["@/src/Db/Db.py:17"], ["Db.py line 17"]), # path compression + (["%s:1" % __file__], ["TestDebug.py line 1"]), + (["@/plugins/Chart/ChartDb.py:100"], ["ChartDb.py line 100"]), # plugins + (["@/main.py:17"], ["main.py line 17"]), # root + (["@\\src\\Db\\__init__.py:17"], ["Db/__init__.py line 17"]), # Windows paths + ([":1"], []), # importlib builtins + ([":1"], []), # importlib builtins + (["/home/ivanq/ZeroNet/src/main.py:13"], ["?/src/main.py line 13"]), # best-effort anonymization + (["C:\\ZeroNet\\core\\src\\main.py:13"], ["?/src/main.py line 13"]), + (["/root/main.py:17"], ["/root/main.py line 17"]), + (["{gevent}:13"], ["/__init__.py line 13"]), # modules + (["{os}:13"], [" line 13"]), # python builtin modules, + (["@/src/Db/Db.py:17", "@/src/Db/DbQuery.py:1"], ["Db.py line 17", "DbQuery.py line 1"]), # mutliple args + (["@/src/Db/Db.py:17", "@/src/Db/Db.py:1"], ["Db.py line 17", "1"]), # same file + (["{os}:1", "@/src/Db/Db.py:17"], [" line 1", "Db.py line 17"]), # builtins + (["{gevent}:1"] + ["{os}:3"] * 4 + ["@/src/Db/Db.py:17"], ["/__init__.py line 1", "...", "Db.py line 17"]), + ]) + def testFormatTraceback(self, items, expected): + q_items = [] + for item in items: + file, line = item.rsplit(":", 1) + if file.startswith("@"): + file = Debug.root_dir + file[1:] + file = file.replace("{os}", os.__file__) + file = file.replace("{gevent}", gevent.__file__) + q_items.append((file, int(line))) + assert Debug.formatTraceback(q_items) == expected + + + def testFormatException(self): + try: + raise ValueError("Test exception") + except: + assert Debug.formatException() == "ValueError: Test exception in TestDebug.py line 42" + try: + os.path.abspath(1) + except: + assert Debug.formatException().startswith("TypeError: expected str, bytes or os.PathLike object, not int in TestDebug.py line 46 > line ") + + + def testFormatStack(self): + assert Debug.formatStack().startswith("TestDebug.py line 52 > <_pytest>/python.py line ") \ No newline at end of file From 6770b450b326e7fba49f25f7b19706ee7eff0cee Mon Sep 17 00:00:00 2001 From: Ivanq Date: Tue, 13 Oct 2020 19:50:08 +0300 Subject: [PATCH 2/3] Handle src/gevent/... paths --- src/Debug/Debug.py | 6 +++++- src/Test/TestDebug.py | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Debug/Debug.py b/src/Debug/Debug.py index dec07919..e6cf28a9 100644 --- a/src/Debug/Debug.py +++ b/src/Debug/Debug.py @@ -47,7 +47,11 @@ def formatTraceback(items, limit=None, fold_builtin=True): is_last = i == len(items) path = path.replace("\\", "/") - if path in ("", ""): + if path.startswith("src/gevent/"): + file_title = "/" + path[len("src/gevent/"):] + is_builtin = True + is_skippable_builtin = False + elif path in ("", ""): file_title = "(importlib)" is_builtin = True is_skippable_builtin = True diff --git a/src/Test/TestDebug.py b/src/Test/TestDebug.py index 695a81a9..f65c13d9 100644 --- a/src/Test/TestDebug.py +++ b/src/Test/TestDebug.py @@ -19,11 +19,12 @@ class TestDebug: (["C:\\ZeroNet\\core\\src\\main.py:13"], ["?/src/main.py line 13"]), (["/root/main.py:17"], ["/root/main.py line 17"]), (["{gevent}:13"], ["/__init__.py line 13"]), # modules - (["{os}:13"], [" line 13"]), # python builtin modules, + (["{os}:13"], [" line 13"]), # python builtin modules + (["src/gevent/event.py:17"], ["/event.py line 17"]), # gevent-overriden __file__ (["@/src/Db/Db.py:17", "@/src/Db/DbQuery.py:1"], ["Db.py line 17", "DbQuery.py line 1"]), # mutliple args (["@/src/Db/Db.py:17", "@/src/Db/Db.py:1"], ["Db.py line 17", "1"]), # same file (["{os}:1", "@/src/Db/Db.py:17"], [" line 1", "Db.py line 17"]), # builtins - (["{gevent}:1"] + ["{os}:3"] * 4 + ["@/src/Db/Db.py:17"], ["/__init__.py line 1", "...", "Db.py line 17"]), + (["{gevent}:1"] + ["{os}:3"] * 4 + ["@/src/Db/Db.py:17"], ["/__init__.py line 1", "...", "Db.py line 17"]) ]) def testFormatTraceback(self, items, expected): q_items = [] @@ -41,12 +42,12 @@ class TestDebug: try: raise ValueError("Test exception") except: - assert Debug.formatException() == "ValueError: Test exception in TestDebug.py line 42" + assert Debug.formatException() == "ValueError: Test exception in TestDebug.py line 43" try: os.path.abspath(1) except: - assert Debug.formatException().startswith("TypeError: expected str, bytes or os.PathLike object, not int in TestDebug.py line 46 > line ") + assert Debug.formatException().startswith("TypeError: expected str, bytes or os.PathLike object, not int in TestDebug.py line 47 > line ") def testFormatStack(self): - assert Debug.formatStack().startswith("TestDebug.py line 52 > <_pytest>/python.py line ") \ No newline at end of file + assert Debug.formatStack().startswith("TestDebug.py line 53 > <_pytest>/python.py line ") \ No newline at end of file From dd08b89c813a1d0f7e3b42c319f93397416aadb0 Mon Sep 17 00:00:00 2001 From: Ivanq Date: Tue, 13 Oct 2020 20:36:39 +0300 Subject: [PATCH 3/3] Make tests pass on Python 3.5 --- src/Test/TestDebug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Test/TestDebug.py b/src/Test/TestDebug.py index f65c13d9..f7708c78 100644 --- a/src/Test/TestDebug.py +++ b/src/Test/TestDebug.py @@ -46,7 +46,7 @@ class TestDebug: try: os.path.abspath(1) except: - assert Debug.formatException().startswith("TypeError: expected str, bytes or os.PathLike object, not int in TestDebug.py line 47 > line ") + assert "in TestDebug.py line 47 > line " in Debug.formatException() def testFormatStack(self):