diff --git a/tests/test_commands.py b/tests/test_commands.py
index eca7fa6..1d51053 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -26,7 +26,9 @@ import pathlib
 import shutil
 import json
 import yaml
+import pytest
 from kas import kas
+from kas.kasusererror import CommandExecError
 
 
 def test_for_all_repos(changedir, tmpdir):
@@ -61,6 +63,14 @@ def test_checkout(changedir, tmpdir):
     assert not os.path.exists('build/sstate-cache')
 
 
+def test_invalid_checkout(changedir, tmpdir, capsys):
+    tdir = str(tmpdir / 'test_commands')
+    shutil.copytree('tests/test_commands', tdir)
+    os.chdir(tdir)
+    with pytest.raises(CommandExecError):
+        kas.kas(['checkout', 'test-invalid.yml'])
+
+
 def test_checkout_create_refs(changedir, tmpdir):
     tdir = str(tmpdir / 'test_commands')
     repo_cache = pathlib.Path(str(tmpdir.mkdir('repos')))
diff --git a/tests/test_commands/test-invalid.yml b/tests/test_commands/test-invalid.yml
new file mode 100644
index 0000000..79004ab
--- /dev/null
+++ b/tests/test_commands/test-invalid.yml
@@ -0,0 +1,7 @@
+header:
+  version: 8
+
+repos:
+  kas_invalid:
+    url: https://example.com/kas.git
+    refspec: this-ref-is-invalid