diff --git a/d2ir/compile.go b/d2ir/compile.go
index e59cb51be..182ebafb4 100644
--- a/d2ir/compile.go
+++ b/d2ir/compile.go
@@ -38,7 +38,8 @@ func (c *compiler) compileScenarios(m *Map) {
 	}
 
 	for _, sf := range scenarios.Fields {
-		if sf.Map() == nil {
+		if sf.Map() == nil || sf.Primary() != nil {
+			c.errorf(sf.References[0].Context.Key, "invalid scenario")
 			continue
 		}
 		base := m.CopyBase(sf)
@@ -59,14 +60,16 @@ func (c *compiler) compileSteps(m *Map) {
 		return
 	}
 	for i, sf := range steps.Fields {
-		if sf.Map() == nil {
+		if sf.Map() == nil || sf.Primary() != nil {
+			c.errorf(sf.References[0].Context.Key, "invalid step")
 			continue
 		}
 		var base *Map
 		if i == 0 {
 			base = m.CopyBase(sf)
 		} else {
-			if steps.Fields[i-1].Map() == nil {
+			if steps.Fields[i-1].Map() == nil || steps.Fields[i-1].Primary() != nil {
+				c.errorf(steps.Fields[i-1].References[0].Context.Key, "invalid step")
 				continue
 			}
 			base = steps.Fields[i-1].Map().CopyBase(sf)
diff --git a/d2ir/compile_test.go b/d2ir/compile_test.go
index bd21d9ae2..1a25e7b8f 100644
--- a/d2ir/compile_test.go
+++ b/d2ir/compile_test.go
@@ -420,6 +420,23 @@ steps: {
 				assertQuery(t, m, 0, 0, nil, "steps.nuclear.quiche")
 			},
 		},
+		{
+			name: "steps_panic",
+			run: func(t testing.TB) {
+				_, err := compile(t, `steps: {
+  shape: sql_table
+  id: int {constraint: primary_key}
+}
+scenarios: {
+  shape: sql_table
+  hey: int {constraint: primary_key}
+}`)
+				assert.ErrorString(t, err, `TestCompile/steps/steps_panic.d2:6:3: invalid scenario
+TestCompile/steps/steps_panic.d2:7:3: invalid scenario
+TestCompile/steps/steps_panic.d2:2:3: invalid step
+TestCompile/steps/steps_panic.d2:3:3: invalid step`)
+			},
+		},
 		{
 			name: "recursive",
 			run: func(t testing.TB) {
diff --git a/e2etests/testdata/todo/steps_panic/dagre/board.exp.json b/e2etests/testdata/todo/steps_panic/dagre/board.exp.json
deleted file mode 100644
index b230f4a99..000000000
--- a/e2etests/testdata/todo/steps_panic/dagre/board.exp.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "",
-  "fontFamily": "SourceSansPro",
-  "shapes": [],
-  "connections": [],
-  "scenarios": [
-    {
-      "name": "hey",
-      "fontFamily": "SourceSansPro",
-      "shapes": [],
-      "connections": []
-    }
-  ],
-  "steps": [
-    {
-      "name": "id",
-      "fontFamily": "SourceSansPro",
-      "shapes": [],
-      "connections": []
-    }
-  ]
-}
diff --git a/e2etests/testdata/todo/steps_panic/dagre/sketch.exp.svg b/e2etests/testdata/todo/steps_panic/dagre/sketch.exp.svg
deleted file mode 100644
index 5169c50f1..000000000
--- a/e2etests/testdata/todo/steps_panic/dagre/sketch.exp.svg
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<svg
-id="d2-svg"
-style="background: white;"
-xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-width="200" height="200" viewBox="-100 -100 200 200"><style type="text/css">
-<![CDATA[
-.shape {
-  shape-rendering: geometricPrecision;
-  stroke-linejoin: round;
-}
-.connection {
-  stroke-linecap: round;
-  stroke-linejoin: round;
-}
-.blend {
-  mix-blend-mode: multiply;
-  opacity: 0.5;
-}
-
-]]>
-</style><script type="application/javascript"><![CDATA[window.addEventListener("DOMContentLoaded", () => {
-  if (document.documentElement.getAttribute("id") !== "d2-svg") {
-    return;
-  }
-  const svgEl = document.documentElement;
-  let width = parseInt(svgEl.getAttribute("width"), 10);
-  let height = parseInt(svgEl.getAttribute("height"), 10);
-  let ratio;
-  if (width > height) {
-    if (width > window.innerWidth) {
-      ratio = window.innerWidth / width;
-    }
-  } else if (height > window.innerHeight) {
-    ratio = window.innerHeight / height;
-  }
-  if (ratio) {
-    svgEl.setAttribute("width", width * ratio - 16);
-    svgEl.setAttribute("height", height * ratio - 16);
-  }
-});
-]]></script><mask id="121760133" maskUnits="userSpaceOnUse" x="-100" y="-100" width="200" height="200">
-<rect x="-100" y="-100" width="200" height="200" fill="white"></rect>
-
-</mask><style type="text/css"><![CDATA[]]></style></svg>
\ No newline at end of file
diff --git a/e2etests/testdata/todo/steps_panic/elk/board.exp.json b/e2etests/testdata/todo/steps_panic/elk/board.exp.json
deleted file mode 100644
index b230f4a99..000000000
--- a/e2etests/testdata/todo/steps_panic/elk/board.exp.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "name": "",
-  "fontFamily": "SourceSansPro",
-  "shapes": [],
-  "connections": [],
-  "scenarios": [
-    {
-      "name": "hey",
-      "fontFamily": "SourceSansPro",
-      "shapes": [],
-      "connections": []
-    }
-  ],
-  "steps": [
-    {
-      "name": "id",
-      "fontFamily": "SourceSansPro",
-      "shapes": [],
-      "connections": []
-    }
-  ]
-}
diff --git a/e2etests/testdata/todo/steps_panic/elk/sketch.exp.svg b/e2etests/testdata/todo/steps_panic/elk/sketch.exp.svg
deleted file mode 100644
index 5169c50f1..000000000
--- a/e2etests/testdata/todo/steps_panic/elk/sketch.exp.svg
+++ /dev/null
@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<svg
-id="d2-svg"
-style="background: white;"
-xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-width="200" height="200" viewBox="-100 -100 200 200"><style type="text/css">
-<![CDATA[
-.shape {
-  shape-rendering: geometricPrecision;
-  stroke-linejoin: round;
-}
-.connection {
-  stroke-linecap: round;
-  stroke-linejoin: round;
-}
-.blend {
-  mix-blend-mode: multiply;
-  opacity: 0.5;
-}
-
-]]>
-</style><script type="application/javascript"><![CDATA[window.addEventListener("DOMContentLoaded", () => {
-  if (document.documentElement.getAttribute("id") !== "d2-svg") {
-    return;
-  }
-  const svgEl = document.documentElement;
-  let width = parseInt(svgEl.getAttribute("width"), 10);
-  let height = parseInt(svgEl.getAttribute("height"), 10);
-  let ratio;
-  if (width > height) {
-    if (width > window.innerWidth) {
-      ratio = window.innerWidth / width;
-    }
-  } else if (height > window.innerHeight) {
-    ratio = window.innerHeight / height;
-  }
-  if (ratio) {
-    svgEl.setAttribute("width", width * ratio - 16);
-    svgEl.setAttribute("height", height * ratio - 16);
-  }
-});
-]]></script><mask id="121760133" maskUnits="userSpaceOnUse" x="-100" y="-100" width="200" height="200">
-<rect x="-100" y="-100" width="200" height="200" fill="white"></rect>
-
-</mask><style type="text/css"><![CDATA[]]></style></svg>
\ No newline at end of file
diff --git a/e2etests/todo_test.go b/e2etests/todo_test.go
index bd2aeded0..5d5953cdd 100644
--- a/e2etests/todo_test.go
+++ b/e2etests/todo_test.go
@@ -159,18 +159,6 @@ small code: |go
 	width: 4
 	height: 3
 }
-`,
-		},
-		{
-			name: "steps_panic",
-			script: `steps: {
-  shape: sql_table
-  id: int {constraint: primary_key}
-}
-scenarios: {
-  shape: sql_table
-  hey: int {constraint: primary_key}
-}
 `,
 		},
 		{