diff --git a/d2graph/layout.go b/d2graph/layout.go
index f1c4a7ecc..870c0efae 100644
--- a/d2graph/layout.go
+++ b/d2graph/layout.go
@@ -305,6 +305,33 @@ func (obj *Object) GetMargin() geo.Spacing {
 		case label.OutsideRightTop, label.OutsideRightMiddle, label.OutsideRightBottom:
 			margin.Right = labelWidth
 		}
+
+		// if an outside label is larger than the object add margin accordingly
+		if labelWidth > obj.Width {
+			dx := labelWidth - obj.Width
+			switch position {
+			case label.OutsideTopLeft, label.OutsideBottomLeft:
+				// label fixed at left will overflow on right
+				margin.Right = dx
+			case label.OutsideTopCenter, label.OutsideBottomCenter:
+				margin.Left = math.Ceil(dx / 2)
+				margin.Right = math.Ceil(dx / 2)
+			case label.OutsideTopRight, label.OutsideBottomRight:
+				margin.Left = dx
+			}
+		}
+		if labelHeight > obj.Height {
+			dy := labelHeight - obj.Height
+			switch position {
+			case label.OutsideLeftTop, label.OutsideRightTop:
+				margin.Bottom = dy
+			case label.OutsideLeftMiddle, label.OutsideRightMiddle:
+				margin.Top = math.Ceil(dy / 2)
+				margin.Bottom = math.Ceil(dy / 2)
+			case label.OutsideLeftBottom, label.OutsideRightBottom:
+				margin.Top = dy
+			}
+		}
 	}
 
 	if obj.Icon != nil && obj.IconPosition != nil && obj.Shape.Value != d2target.ShapeImage {
diff --git a/d2layouts/d2grid/layout.go b/d2layouts/d2grid/layout.go
index 9614e9fab..506a9b8a8 100644
--- a/d2layouts/d2grid/layout.go
+++ b/d2layouts/d2grid/layout.go
@@ -142,18 +142,6 @@ func Layout(ctx context.Context, g *d2graph.Graph) error {
 func layoutGrid(g *d2graph.Graph, obj *d2graph.Object) (*gridDiagram, error) {
 	gd := newGridDiagram(obj)
 
-	// to handle objects with outside labels, we adjust their dimensions before layout and
-	// after layout, we remove the label adjustment and reposition TopLeft if needed
-	revertAdjustments := gd.sizeForOutsideLabels()
-
-	if gd.rows != 0 && gd.columns != 0 {
-		gd.layoutEvenly(g, obj)
-	} else {
-		gd.layoutDynamic(g, obj)
-	}
-
-	revertAdjustments()
-
 	// position labels and icons
 	for _, o := range gd.objects {
 		positionedLabel := false
@@ -182,6 +170,18 @@ func layoutGrid(g *d2graph.Graph, obj *d2graph.Object) (*gridDiagram, error) {
 		}
 	}
 
+	// to handle objects with outside labels, we adjust their dimensions before layout and
+	// after layout, we remove the label adjustment and reposition TopLeft if needed
+	revertAdjustments := gd.sizeForOutsideLabels()
+
+	if gd.rows != 0 && gd.columns != 0 {
+		gd.layoutEvenly(g, obj)
+	} else {
+		gd.layoutDynamic(g, obj)
+	}
+
+	revertAdjustments()
+
 	return gd, nil
 }