diff --git a/tools/automation/release/__init__.py b/tools/automation/release/__init__.py
index 5f8c8fd31..c70d6003b 100644
--- a/tools/automation/release/__init__.py
+++ b/tools/automation/release/__init__.py
@@ -88,13 +88,19 @@ def generate_news(session: Session, version: str) -> None:
 def update_version_file(version: str, filepath: str) -> None:
     with open(filepath, "r", encoding="utf-8") as f:
         content = list(f)
+
+    file_modified = False
     with open(filepath, "w", encoding="utf-8") as f:
         for line in content:
             if line.startswith("__version__ ="):
                 f.write('__version__ = "{}"\n'.format(version))
+                file_modified = True
             else:
                 f.write(line)
 
+    assert file_modified, \
+           "Version file {} did not get modified".format(filepath)
+
 
 def create_git_tag(session: Session, tag_name: str, *, message: str) -> None:
     session.run(