diff --git a/d2graph/d2graph.go b/d2graph/d2graph.go
index 76046e377..409e59cf0 100644
--- a/d2graph/d2graph.go
+++ b/d2graph/d2graph.go
@@ -584,7 +584,8 @@ func (obj *Object) Text() *d2target.MText {
 	}
 
 	if obj.OuterSequenceDiagram() == nil {
-		if obj.IsContainer() && obj.Shape.Value != "text" {
+		// Note: during grid layout when children are temporarily removed `IsContainer` is false
+		if (obj.IsContainer() || obj.IsGridDiagram()) && obj.Shape.Value != "text" {
 			fontSize = obj.Level().LabelSize()
 		}
 	} else {