d2compiler: Integrate d2ir (wip)

d2compiler/compile.go

@@ -105,7 +105,14 @@ func (c *compiler) errorf(n d2ast.Node, f string, v ...interface{}) {
 }
 
 func (c *compiler) compileMap(obj *d2graph.Object, m *d2ir.Map) {
+	shape := m.GetField("shape")
+	if shape != nil {
+		c.compileField(obj, shape)
+	}
 	for _, f := range m.Fields {
+		if f.Name == "shape" {
+			continue
+		}
 		c.compileField(obj, f)
 	}
 
@@ -135,7 +142,7 @@ func (c *compiler) compileField(obj *d2graph.Object, f *d2ir.Field) {
 		return
 	}
 
-	obj = obj.EnsureChild([]string{f.Name})
+	obj = obj.EnsureChild(d2graphIDA([]string{f.Name}))
 	if f.Primary() != nil {
 		c.compileLabel(obj.Attributes, f)
 	}
@@ -143,15 +150,20 @@ func (c *compiler) compileField(obj *d2graph.Object, f *d2ir.Field) {
 		c.compileMap(obj, f.Map())
 	}
 
-	for _, er := range f.References {
+	for _, fr := range f.References {
+		if fr.OurValue() && fr.Context.Key.Value.Map != nil {
+			obj.Map = fr.Context.Key.Value.Map
+		}
+		scopeObjIDA := d2ir.IDA(fr.Context.ScopeMap)
+		scopeObj, _ := obj.Graph.Root.HasChild(scopeObjIDA)
 		obj.References = append(obj.References, d2graph.Reference{
-			Key: er.KeyPath,
+			Key:          fr.KeyPath,
-			KeyPathIndex: er.KeyPathIndex(),
+			KeyPathIndex: fr.KeyPathIndex(),
 
-			MapKey: er.Context.Key,
+			MapKey:          fr.Context.Key,
-			MapKeyEdgeIndex: er.Context.EdgeIndex(),
+			MapKeyEdgeIndex: fr.Context.EdgeIndex(),
-			Scope: er.Context.Scope,
+			Scope:           fr.Context.Scope,
-			ScopeObj: obj.Parent,
+			ScopeObj:        scopeObj,
 		})
 	}
 }
@@ -181,6 +193,9 @@ func (c *compiler) compileLabel(attrs *d2graph.Attributes, f d2ir.Node) {
 
 func (c *compiler) compileReserved(attrs *d2graph.Attributes, f *d2ir.Field) {
 	if f.Primary() == nil {
+		if f.Composite != nil {
+			c.errorf(f.LastPrimaryKey(), "reserved field %v does not accept composite", f.Name)
+		}
 		return
 	}
 	scalar := f.Primary().Value
@@ -244,7 +259,12 @@ func (c *compiler) compileReserved(attrs *d2graph.Attributes, f *d2ir.Field) {
 		attrs.Direction.Value = scalar.ScalarString()
 		attrs.Direction.MapKey = f.LastPrimaryKey()
 	case "constraint":
-		// Compilation for shape-specific keywords happens elsewhere
+		if _, ok := scalar.(d2ast.String); !ok {
+			c.errorf(f.LastPrimaryKey(), "constraint value must be a string")
+			return
+		}
+		attrs.Constraint.Value = scalar.ScalarString()
+		attrs.Constraint.MapKey = f.LastPrimaryKey()
 	}
 }
 
@@ -311,7 +331,7 @@ func compileStyleFieldInit(attrs *d2graph.Attributes, f *d2ir.Field) {
 }
 
 func (c *compiler) compileEdge(obj *d2graph.Object, e *d2ir.Edge) {
-	edge, err := obj.Connect(e.ID.SrcPath, e.ID.DstPath, e.ID.SrcArrow, e.ID.DstArrow, "")
+	edge, err := obj.Connect(d2graphIDA(e.ID.SrcPath), d2graphIDA(e.ID.DstPath), e.ID.SrcArrow, e.ID.DstArrow, "")
 	if err != nil {
 		c.errorf(e.References[0].AST(), err.Error())
 		return
@@ -332,12 +352,14 @@ func (c *compiler) compileEdge(obj *d2graph.Object, e *d2ir.Edge) {
 	}
 
 	for _, er := range e.References {
+		scopeObjIDA := d2ir.IDA(er.Context.ScopeMap)
+		scopeObj, _ := edge.Src.Graph.Root.HasChild(scopeObjIDA)
 		edge.References = append(edge.References, d2graph.EdgeReference{
 			Edge:            er.Context.Edge,
 			MapKey:          er.Context.Key,
 			MapKeyEdgeIndex: er.Context.EdgeIndex(),
 			Scope:           er.Context.Scope,
-			ScopeObj: obj,
+			ScopeObj:        scopeObj,
 		})
 	}
 }
@@ -452,6 +474,14 @@ func (c *compiler) compileClass(obj *d2graph.Object) {
 		}
 	}
 
+	for _, ch := range obj.ChildrenArray {
+		for i := 0; i < len(obj.Graph.Objects); i++ {
+			if obj.Graph.Objects[i] == ch {
+				obj.Graph.Objects = append(obj.Graph.Objects[:i], obj.Graph.Objects[i+1:]...)
+				i--
+			}
+		}
+	}
 	obj.Children = nil
 	obj.ChildrenArray = nil
 }
@@ -469,25 +499,20 @@ func (c *compiler) compileSQLTable(obj *d2graph.Object) {
 			Name: d2target.Text{Label: col.IDVal},
 			Type: d2target.Text{Label: typ},
 		}
-		// The only map a sql table field could have is to specify constraint
+		if col.Attributes.Constraint.Value != "" {
-		if col.Map != nil {
+			d2Col.Constraint = col.Attributes.Constraint.Value
-			for _, n := range col.Map.Nodes {
-				if n.MapKey.Key == nil || len(n.MapKey.Key.Path) == 0 {
-					continue
 		}
-				if n.MapKey.Key.Path[0].Unbox().ScalarString() == "constraint" {
-					if n.MapKey.Value.StringBox().Unbox() == nil {
-						c.errorf(n.MapKey, "constraint value must be a string")
-						return
-					}
-					d2Col.Constraint = n.MapKey.Value.StringBox().Unbox().ScalarString()
-				}
-			}
-		}
-
 		obj.SQLTable.Columns = append(obj.SQLTable.Columns, d2Col)
 	}
 
+	for _, ch := range obj.ChildrenArray {
+		for i := 0; i < len(obj.Graph.Objects); i++ {
+			if obj.Graph.Objects[i] == ch {
+				obj.Graph.Objects = append(obj.Graph.Objects[:i], obj.Graph.Objects[i+1:]...)
+				i--
+			}
+		}
+	}
 	obj.Children = nil
 	obj.ChildrenArray = nil
 }
@@ -604,3 +629,13 @@ func init() {
 		FullToShortLanguageAliases[v] = k
 	}
 }
+
+func d2graphIDA(irIDA []string) (ida []string) {
+	for _, el := range irIDA {
+		n := &d2ast.KeyPath{
+			Path: []*d2ast.StringBox{d2ast.MakeValueBox(d2ast.RawString(el, true)).StringBox()},
+		}
+		ida = append(ida, d2format.Format(n))
+	}
+	return ida
+}

d2compiler/compile_test.go

@@ -336,8 +336,7 @@ x: {
 `,
 			assertions: func(t *testing.T, g *d2graph.Graph) {
 				tassert.Equal(t, "y", g.Objects[1].ID)
-				tassert.Equal(t, g.Root.AbsID(), g.Objects[1].References[0].ScopeObj.AbsID())
+				tassert.Equal(t, g.Objects[0].AbsID(), g.Objects[1].References[0].ScopeObj.AbsID())
-				tassert.Equal(t, g.Objects[0].AbsID(), g.Objects[1].References[0].UnresolvedScopeObj.AbsID())
 			},
 		},
 		{
@@ -1847,7 +1846,7 @@ choo: {
     test_id: varchar(64) {constraint: [primary_key, foreign_key]}
 }
 `,
-			expErr: `d2/testdata/d2compiler/TestCompile/sql-panic.d2:3:27: constraint value must be a string`,
+			expErr: `d2/testdata/d2compiler/TestCompile/sql-panic.d2:3:27: reserved field constraint does not accept composite`,
 		},
 		{
 			name: "wrong_column_index",
@@ -1932,7 +1931,7 @@ func testScenarios(t *testing.T) {
 		run  func(t *testing.T)
 	}{
 		{
-			name: "one",
+			name: "root",
 			run: func(t *testing.T) {
 				g := assertCompile(t, `base
 
@@ -1950,6 +1949,34 @@ layers: {
 				assert.JSON(t, "two", g.Layers[1].Name)
 			},
 		},
+		{
+			name: "recursive",
+			run: func(t *testing.T) {
+				g := assertCompile(t, `base
+
+layers: {
+  one: {
+    santa
+  }
+  two: {
+    clause
+		steps: {
+			seinfeld: {
+				reindeer
+			}
+			missoula: {
+				montana
+			}
+		}
+  }
+}
+`, "")
+				assert.Equal(t, 2, len(g.Layers))
+				assert.Equal(t, "one", g.Layers[0].Name)
+				assert.Equal(t, "two", g.Layers[1].Name)
+				assert.Equal(t, 2, len(g.Layers[1].Steps))
+			},
+		},
 	}
 
 	for _, tc := range tca {

d2graph/d2graph.go

@@ -109,6 +109,7 @@ type Attributes struct {
 	Shape Scalar `json:"shape"`
 
 	Direction  Scalar `json:"direction"`
+	Constraint Scalar `json:"constraint"`
 }
 
 // TODO references at the root scope should have their Scope set to root graph AST
@@ -119,9 +120,7 @@ type Reference struct {
 	MapKey          *d2ast.Key `json:"-"`
 	MapKeyEdgeIndex int        `json:"map_key_edge_index"`
 	Scope           *d2ast.Map `json:"-"`
-	// The ScopeObj and UnresolvedScopeObj are the same except when the key contains underscores
 	ScopeObj        *Object    `json:"-"`
-	UnresolvedScopeObj *Object `json:"-"`
 }
 
 func (r Reference) MapKeyEdgeDest() bool {
@@ -517,6 +516,9 @@ func (obj *Object) newObject(id string) *Object {
 }
 
 func (obj *Object) HasChild(ids []string) (*Object, bool) {
+	if len(ids) == 0 {
+		return obj, true
+	}
 	if len(ids) == 1 && ids[0] != "style" {
 		_, ok := ReservedKeywords[ids[0]]
 		if ok {
@@ -632,15 +634,22 @@ func (obj *Object) FindEdges(mk *d2ast.Key) ([]*Edge, bool) {
 	return ea, true
 }
 
-// EnsureChild grabs the child by ids or creates it if it does not exist including all
+func (obj *Object) ensureChildEdge(ids []string) *Object {
-// intermediate nodes.
+	for i := range ids {
-func (obj *Object) EnsureChild(ids []string) *Object {
 		switch obj.Attributes.Shape.Value {
 		case d2target.ShapeClass, d2target.ShapeSQLTable:
 			// This will only be called for connecting edges where we want to truncate to the
 			// container.
 			return obj
 		}
+		obj = obj.EnsureChild(ids[i : i+1])
+	}
+	return obj
+}
+
+// EnsureChild grabs the child by ids or creates it if it does not exist including all
+// intermediate nodes.
+func (obj *Object) EnsureChild(ids []string) *Object {
 	_, is := ReservedKeywordHolders[ids[0]]
 	if len(ids) == 1 && !is {
 		_, ok := ReservedKeywords[ids[0]]
@@ -664,8 +673,7 @@ func (obj *Object) EnsureChild(ids []string) *Object {
 }
 
 func (obj *Object) AppendReferences(ida []string, ref Reference, unresolvedObj *Object) {
-	ref.ScopeObj = obj
+	ref.ScopeObj = unresolvedObj
-	ref.UnresolvedScopeObj = unresolvedObj
 	numUnderscores := 0
 	for i := range ida {
 		if ida[i] == "_" {
@@ -953,8 +961,8 @@ func (obj *Object) Connect(srcID, dstID []string, srcArrow, dstArrow bool, label
 		}
 	}
 
-	src := srcObj.EnsureChild(srcID)
+	src := srcObj.ensureChildEdge(srcID)
-	dst := dstObj.EnsureChild(dstID)
+	dst := dstObj.ensureChildEdge(dstID)
 
 	if src.OuterSequenceDiagram() != dst.OuterSequenceDiagram() {
 		return nil, errors.New("connections within sequence diagrams can connect only to other objects within the same sequence diagram")

d2graph/seqdiagram.go

@@ -65,7 +65,7 @@ func (obj *Object) ContainsAnyObject(objects []*Object) bool {
 
 func (o *Object) ContainedBy(obj *Object) bool {
 	for _, ref := range o.References {
-		curr := ref.UnresolvedScopeObj
+		curr := ref.ScopeObj
 		for curr != nil {
 			if curr == obj {
 				return true

d2ir/compile.go

@@ -25,6 +25,7 @@ func Compile(ast *d2ast.Map) (*Map, error) {
 			}},
 		},
 	}
+	m.parent.(*Field).References[0].Context.ScopeMap = m
 	c.compileMap(m, ast)
 	c.compileScenarios(m)
 	c.compileSteps(m)
@@ -85,9 +86,10 @@ func (c *compiler) compileMap(dst *Map, ast *d2ast.Map) {
 	for _, n := range ast.Nodes {
 		switch {
 		case n.MapKey != nil:
-			c.compileKey(dst, &RefContext{
+			c.compileKey(&RefContext{
 				Key:      n.MapKey,
 				Scope:    ast,
+				ScopeMap: dst,
 			})
 		case n.Substitution != nil:
 			panic("TODO")
@@ -95,11 +97,11 @@ func (c *compiler) compileMap(dst *Map, ast *d2ast.Map) {
 	}
 }
 
-func (c *compiler) compileKey(dst *Map, refctx *RefContext) {
+func (c *compiler) compileKey(refctx *RefContext) {
 	if len(refctx.Key.Edges) == 0 {
-		c.compileField(dst, refctx.Key.Key, refctx)
+		c.compileField(refctx.ScopeMap, refctx.Key.Key, refctx)
 	} else {
-		c.compileEdges(dst, refctx)
+		c.compileEdges(refctx)
 	}
 }
 
@@ -137,9 +139,9 @@ func (c *compiler) compileField(dst *Map, kp *d2ast.KeyPath, refctx *RefContext)
 	}
 }
 
-func (c *compiler) compileEdges(dst *Map, refctx *RefContext) {
+func (c *compiler) compileEdges(refctx *RefContext) {
 	if refctx.Key.Key != nil {
-		f, err := dst.EnsureField(refctx.Key.Key, refctx)
+		f, err := refctx.ScopeMap.EnsureField(refctx.Key.Key, refctx)
 		if err != nil {
 			c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
 			return
@@ -153,7 +155,7 @@ func (c *compiler) compileEdges(dst *Map, refctx *RefContext) {
 				parent: f,
 			}
 		}
-		dst = f.Map()
+		refctx.ScopeMap = f.Map()
 	}
 
 	eida := NewEdgeIDs(refctx.Key)
@@ -163,7 +165,7 @@ func (c *compiler) compileEdges(dst *Map, refctx *RefContext) {
 
 		var e *Edge
 		if eid.Index != nil {
-			ea := dst.GetEdges(eid)
+			ea := refctx.ScopeMap.GetEdges(eid)
 			if len(ea) == 0 {
 				c.errorf(refctx.Edge, "indexed edge does not exist")
 				continue
@@ -172,21 +174,21 @@ func (c *compiler) compileEdges(dst *Map, refctx *RefContext) {
 			e.References = append(e.References, &EdgeReference{
 				Context: refctx,
 			})
-			dst.appendFieldReferences(0, refctx.Edge.Src, refctx)
+			refctx.ScopeMap.appendFieldReferences(0, refctx.Edge.Src, refctx)
-			dst.appendFieldReferences(0, refctx.Edge.Dst, refctx)
+			refctx.ScopeMap.appendFieldReferences(0, refctx.Edge.Dst, refctx)
 		} else {
-			_, err := dst.EnsureField(refctx.Edge.Src, refctx)
+			_, err := refctx.ScopeMap.EnsureField(refctx.Edge.Src, refctx)
 			if err != nil {
 				c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
 				continue
 			}
-			_, err = dst.EnsureField(refctx.Edge.Dst, refctx)
+			_, err = refctx.ScopeMap.EnsureField(refctx.Edge.Dst, refctx)
 			if err != nil {
 				c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
 				continue
 			}
 
-			e, err = dst.CreateEdge(eid, refctx)
+			e, err = refctx.ScopeMap.CreateEdge(eid, refctx)
 			if err != nil {
 				c.err.Errors = append(c.err.Errors, err.(d2ast.Error))
 				continue

d2ir/d2ir.go

@@ -262,17 +262,9 @@ func (f *Field) Copy(newParent Node) Node {
 }
 
 func (f *Field) lastPrimaryRef() *FieldReference {
-	inEdge := ParentEdge(f) != nil
 	for i := len(f.References) - 1; i >= 0; i-- {
-		fr := f.References[i]
+		if f.References[i].OurValue() {
-		if inEdge && len(fr.Context.Key.Edges) > 0 {
+			return f.References[i]
-			if fr.String == fr.Context.Key.EdgeKey.Path[len(fr.Context.Key.EdgeKey.Path)-1].Unbox() {
-				return fr
-			}
-		} else {
-			if fr.String == fr.Context.Key.Key.Path[len(fr.Context.Key.Key.Path)-1].Unbox() {
-				return fr
-			}
 		}
 	}
 	return nil
@@ -470,6 +462,17 @@ type FieldReference struct {
 	Context *RefContext `json:"context"`
 }
 
+// OurValue returns true if the Value in Context.Key.Value corresponds to the Field
+// represented by String.
+func (fr *FieldReference) OurValue() bool {
+	if fr.KeyPath == fr.Context.Key.Key {
+		return fr.KeyPathIndex() == len(fr.KeyPath.Path)-1
+	} else if fr.KeyPath == fr.Context.Key.EdgeKey {
+		return fr.KeyPathIndex() == len(fr.KeyPath.Path)-1
+	}
+	return false
+}
+
 func (fr *FieldReference) KeyPathIndex() int {
 	for i, sb := range fr.KeyPath.Path {
 		if sb.Unbox() == fr.String {
@@ -507,6 +510,7 @@ type RefContext struct {
 	Edge     *d2ast.Edge `json:"edge"`
 	Key      *d2ast.Key  `json:"key"`
 	Scope    *d2ast.Map  `json:"-"`
+	ScopeMap *Map        `json:"-"`
 }
 
 func (rc *RefContext) Copy() *RefContext {
@@ -514,26 +518,6 @@ func (rc *RefContext) Copy() *RefContext {
 	return &tmp
 }
 
-// UnresolvedScopeMap is scope prior to interpreting _
-// It does this by finding the referenced *Map of rc.Scope
-func (rc *RefContext) UnresolvedScopeMap(m *Map) *Map {
-	for {
-		fm := ParentField(m)
-		if fm == nil {
-			return m
-		}
-		for _, ref := range fm.References {
-			if ref.KeyPath != ref.Context.Key.Key {
-				continue
-			}
-			if ref.Context.Key.Value.Unbox() == rc.Scope {
-				return m
-			}
-		}
-		m = ParentMap(m)
-	}
-}
-
 func (rc *RefContext) EdgeIndex() int {
 	for i, e := range rc.Key.Edges {
 		if e == rc.Edge {
@@ -991,3 +975,20 @@ func parentPrimaryKey(n Node) *d2ast.Key {
 	}
 	return nil
 }
+
+func IDA(n Node) (ida []string) {
+	for {
+		f, ok := n.(*Field)
+		if ok {
+			if f.Name == "" {
+				return ida
+			}
+			ida = append(ida, f.Name)
+		}
+		f = ParentField(n)
+		if f == nil {
+			return ida
+		}
+		n = f
+	}
+}

d2layouts/d2sequence/sequence_diagram.go

@@ -225,7 +225,7 @@ func (sd *sequenceDiagram) placeGroup(group *d2graph.Object) {
 	for _, n := range sd.notes {
 		inGroup := false
 		for _, ref := range n.References {
-			curr := ref.UnresolvedScopeObj
+			curr := ref.ScopeObj
 			for curr != nil {
 				if curr == group {
 					inGroup = true

d2oracle/edit.go

@@ -1104,7 +1104,7 @@ func move(g *d2graph.Graph, key, newKey string) (*d2graph.Graph, error) {
 				Key:    detachedMK.Key,
 				MapKey: detachedMK,
 				Scope:  mostNestedRef.Scope,
-			}, mostNestedRef.UnresolvedScopeObj)
+			}, mostNestedRef.ScopeObj)
 		}
 	}
 
@@ -1279,8 +1279,8 @@ func move(g *d2graph.Graph, key, newKey string) (*d2graph.Graph, error) {
 
 		// We don't want this to be underscore-resolved scope. We want to ignore underscores
 		var scopeak []string
-		if ref.UnresolvedScopeObj != g.Root {
+		if ref.ScopeObj != g.Root {
-			scopek, err := d2parser.ParseKey(ref.UnresolvedScopeObj.AbsID())
+			scopek, err := d2parser.ParseKey(ref.ScopeObj.AbsID())
 			if err != nil {
 				return nil, err
 			}