diff --git a/bin/check-test-version.py b/bin/check-test-version.py
index 723bca0ba2c..6105a3e2653 100755
--- a/bin/check-test-version.py
+++ b/bin/check-test-version.py
@@ -323,5 +323,9 @@ def check_test_version(
         if opts.ignore and exercise in opts.ignore:
             continue
         if os.path.isdir(os.path.join('exercises', exercise)):
-            result = check_test_version(exercise, **kwargs) and result
+            try:
+                result = check_test_version(exercise, **kwargs) and result
+            except FileNotFoundError as e:
+                print(str(e))
+                result = False
     sys.exit(0 if result else 1)