diff --git a/scripts/update_bugs_by_version.py b/scripts/update_bugs_by_version.py
index 56daf1513..442cd0fdc 100755
--- a/scripts/update_bugs_by_version.py
+++ b/scripts/update_bugs_by_version.py
@@ -8,7 +8,6 @@
 
 import json
 import re
-import sys
 from pathlib import Path
 
 def comp(version_string):
@@ -35,7 +34,9 @@ for key, value in versions.items():
             continue
         value['bugs'] += [bug['name']]
 
-new_contents = json.dumps(versions, sort_keys=True, indent=4, separators=(',', ': '))
-old_contents = (root_path / 'docs/bugs_by_version.json').read_text(encoding='utf8')
-(root_path / 'docs/bugs_by_version.json').write_text(new_contents, encoding='utf8')
-sys.exit(old_contents != new_contents)
+(root_path / 'docs/bugs_by_version.json').write_text(json.dumps(
+    versions,
+    sort_keys=True,
+    indent=4,
+    separators=(',', ': ')
+), encoding='utf8')
diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh
index 250d10a6e..79e9d64a5 100755
--- a/test/cmdlineTests.sh
+++ b/test/cmdlineTests.sh
@@ -355,7 +355,7 @@ function test_via_ir_equivalence()
 
     for yul_file in $(find . -name "${output_file_prefix}*.yul" | sort -V); do
         bin_output_two_stage+=$(
-	    msg_on_error --no-stderr "$SOLC" --strict-assembly --bin "${optimizer_flags[@]}" "$yul_file" |
+        msg_on_error --no-stderr "$SOLC" --strict-assembly --bin "${optimizer_flags[@]}" "$yul_file" |
                 sed '/^Binary representation:$/d' |
                 sed '/^=======/d'
         )
@@ -375,8 +375,13 @@ function test_via_ir_equivalence()
 
 ## RUN
 
-echo "Checking that the bug list is up to date..."
-"$REPO_ROOT"/scripts/update_bugs_by_version.py
+SOLTMPDIR=$(mktemp -d)
+printTask "Checking that the bug list is up to date..."
+cp "${REPO_ROOT}/docs/bugs_by_version.json" "${SOLTMPDIR}/original_bugs_by_version.json"
+"${REPO_ROOT}/scripts/update_bugs_by_version.py"
+diff --unified "${SOLTMPDIR}/original_bugs_by_version.json" "${REPO_ROOT}/docs/bugs_by_version.json" || \
+    fail "The bug list in bugs_by_version.json was out of date and has been updated. Please investigate and submit a bugfix if necessary."
+rm -r "$SOLTMPDIR"
 
 printTask "Testing unknown options..."
 (