diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 3883537f4..0f9f15eb3 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -16,7 +16,7 @@ jobs: with: go-version-file: ./go.mod cache: true - - run: COLOR=1 CI_FORCE=1 ./make.sh all race + - run: DAILY=1 COLOR=1 CI_FORCE=1 ./make.sh all race env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} diff --git a/ci/release/changelogs/next.md b/ci/release/changelogs/next.md index 69969d43e..f3c0d2a77 100644 --- a/ci/release/changelogs/next.md +++ b/ci/release/changelogs/next.md @@ -1,21 +1,5 @@ #### Features 🚀 -- Configure timeout value with D2_TIMEOUT env var [#1392](https://github.com/terrastruct/d2/pull/1392) -- Scale renders and disable fit to screen with `--scale` flag [#1413](https://github.com/terrastruct/d2/pull/1413) -- `null` keyword can be used to un-declare. See [docs](https://d2lang.com/tour/TODO) [#1446](https://github.com/terrastruct/d2/pull/1446) - #### Improvements 🧹 -- Display version on CLI help invocation [#1400](https://github.com/terrastruct/d2/pull/1400) -- Improved readability of connection labels when they overlap another connection [#447](https://github.com/terrastruct/d2/pull/447) -- Error message when `shape` is given a composite [#1415](https://github.com/terrastruct/d2/pull/1415) -- Improved rendering and text measurement for code shapes [#1425](https://github.com/terrastruct/d2/pull/1425) -- The autoformatter moves board declarations to the bottom of its scope [#1424](https://github.com/terrastruct/d2/pull/1424) -- All font styles in sketch mode use a consistent font-family [#1463](https://github.com/terrastruct/d2/pull/1463) -- Tooltip and link icons are now positioned on shape border [#1466](https://github.com/terrastruct/d2/pull/1466) -- Tooltip and link icons are always rendered over shapes [#1467](https://github.com/terrastruct/d2/pull/1467) - #### Bugfixes ⛑️ - -- Fixes edge case in compiler using dots in quotes [#1401](https://github.com/terrastruct/d2/pull/1401) -- Fixes grid label font size for TALA [#1412](https://github.com/terrastruct/d2/pull/1412) diff --git a/ci/release/changelogs/v0.6.0.md b/ci/release/changelogs/v0.6.0.md new file mode 100644 index 000000000..16c5d4747 --- /dev/null +++ b/ci/release/changelogs/v0.6.0.md @@ -0,0 +1,54 @@ +D2 v0.6 introduces variable substitutions and globs. These two were the last of the features planned in the initial designs for D2, and v1 is now very close! + +The power of variables and globs in a programming language need no introduction, so here's two minimal examples to get started: + +**Variables**: +```d2 +vars: { + color: aquamarine +} + +x.style.fill: ${color} +``` + +**Globs**: +```d2 +x +y +z + +*.style.fill: aquamarine +``` + +Both are live on [D2 Playground](https://play.d2lang.com) so give it a try! Looking forward to your issues and iterating + +Layout capability also takes a subtle but important step forward: you can now customize the position of labels and icons. + +#### Features 🚀 + +- Variables and substitutions are implemented. See [docs](https://d2lang.com/tour/vars). [#1473](https://github.com/terrastruct/d2/pull/1473) +- Configure timeout value with D2_TIMEOUT env var [#1392](https://github.com/terrastruct/d2/pull/1392) +- Scale renders and disable fit to screen with `--scale` flag [#1413](https://github.com/terrastruct/d2/pull/1413) +- `null` keyword can be used to un-declare. See [docs](https://d2lang.com/tour/overrides#null) [#1446](https://github.com/terrastruct/d2/pull/1446) +- Develop multi-board diagrams in watch mode (links to layers/scenarios/steps work in `--watch`) [#1503](https://github.com/terrastruct/d2/pull/1503) +- Glob patterns have been implemented. See [docs](https://d2lang.com/tour/globs). [#1479](https://github.com/terrastruct/d2/pull/1479) +- Ampersand filters have been implemented. See [docs](https://d2lang.com/tour/filters). [#1509](https://github.com/terrastruct/d2/pull/1509) + +#### Improvements 🧹 + +- Display version on CLI help invocation [#1400](https://github.com/terrastruct/d2/pull/1400) +- Improved readability of connection labels when they overlap another connection [#447](https://github.com/terrastruct/d2/pull/447) +- Error message when `shape` is given a composite [#1415](https://github.com/terrastruct/d2/pull/1415) +- Improved rendering and text measurement for code shapes [#1425](https://github.com/terrastruct/d2/pull/1425) +- The autoformatter moves board declarations to the bottom of its scope [#1424](https://github.com/terrastruct/d2/pull/1424) +- All font styles in sketch mode use a consistent font-family [#1463](https://github.com/terrastruct/d2/pull/1463) +- Tooltip and link icons are positioned on shape border [#1466](https://github.com/terrastruct/d2/pull/1466) +- Tooltip and link icons are always rendered over shapes [#1467](https://github.com/terrastruct/d2/pull/1467) +- Boards with no objects are considered folders [#1504](https://github.com/terrastruct/d2/pull/1504) +- `DEBUG` environment variable ignored if set incorrectly [#1505](https://github.com/terrastruct/d2/pull/1505) + +#### Bugfixes ⛑️ + +- Fixes edge case in compiler using dots in quotes [#1401](https://github.com/terrastruct/d2/pull/1401) +- Fixes grid label font size for TALA [#1412](https://github.com/terrastruct/d2/pull/1412) +- Fixes person shape label positioning with `multiple` or `3d` [#1478](https://github.com/terrastruct/d2/pull/1478) diff --git a/ci/release/docker/Dockerfile b/ci/release/docker/Dockerfile index 0d7789a26..b534b02ff 100644 --- a/ci/release/docker/Dockerfile +++ b/ci/release/docker/Dockerfile @@ -1,5 +1,5 @@ # https://hub.docker.com/repository/docker/terrastruct/d2 -FROM debian:latest +FROM ubuntu:latest ARG TARGETARCH diff --git a/d2ast/d2ast.go b/d2ast/d2ast.go index a5be3fa98..fdbe9c9c3 100644 --- a/d2ast/d2ast.go +++ b/d2ast/d2ast.go @@ -500,6 +500,19 @@ type Number struct { type UnquotedString struct { Range Range `json:"range"` Value []InterpolationBox `json:"value"` + // Pattern holds the parsed glob pattern if in a key and the unquoted string represents a valid pattern. + Pattern []string `json:"pattern,omitempty"` +} + +func (s *UnquotedString) Coalesce() { + var b strings.Builder + for _, box := range s.Value { + if box.String == nil { + break + } + b.WriteString(*box.String) + } + s.SetString(b.String()) } func FlatUnquotedString(s string) *UnquotedString { @@ -513,6 +526,17 @@ type DoubleQuotedString struct { Value []InterpolationBox `json:"value"` } +func (s *DoubleQuotedString) Coalesce() { + var b strings.Builder + for _, box := range s.Value { + if box.String == nil { + break + } + b.WriteString(*box.String) + } + s.SetString(b.String()) +} + func FlatDoubleQuotedString(s string) *DoubleQuotedString { return &DoubleQuotedString{ Value: []InterpolationBox{{String: &s}}, @@ -586,7 +610,7 @@ func (m *Map) IsFileMap() bool { type Key struct { Range Range `json:"range"` - // Indicates this MapKey is an override selector. + // Indicates this MapKey is a filter selector. Ampersand bool `json:"ampersand,omitempty"` // At least one of Key and Edges will be set but all four can also be set. @@ -696,6 +720,19 @@ func (mk *Key) SetScalar(scalar ScalarBox) { } } +func (mk *Key) HasQueryGlob() bool { + if mk.Key.HasGlob() && len(mk.Edges) == 0 { + return true + } + if mk.EdgeIndex != nil && mk.EdgeIndex.Glob && mk.EdgeKey == nil { + return true + } + if mk.EdgeKey.HasGlob() { + return true + } + return false +} + type KeyPath struct { Range Range `json:"range"` Path []*StringBox `json:"path"` @@ -716,6 +753,37 @@ func (kp *KeyPath) IDA() (ida []string) { return ida } +func (kp *KeyPath) Copy() *KeyPath { + kp2 := *kp + kp2.Path = nil + kp2.Path = append(kp2.Path, kp.Path...) + return &kp2 +} + +func (kp *KeyPath) HasDoubleGlob() bool { + if kp == nil { + return false + } + for _, el := range kp.Path { + if el.UnquotedString != nil && el.ScalarString() == "**" { + return true + } + } + return false +} + +func (kp *KeyPath) HasGlob() bool { + if kp == nil { + return false + } + for _, el := range kp.Path { + if el.UnquotedString != nil && len(el.UnquotedString.Pattern) > 0 { + return true + } + } + return false +} + type Edge struct { Range Range `json:"range"` @@ -1034,6 +1102,10 @@ func (sb *StringBox) Unbox() String { } } +func (sb *StringBox) ScalarString() string { + return sb.Unbox().ScalarString() +} + // InterpolationBox is used to select between strings and substitutions in unquoted and // double quoted strings. There is no corresponding interface to avoid unnecessary // abstraction. @@ -1046,12 +1118,11 @@ type InterpolationBox struct { // & is only special if it begins a key. // - is only special if followed by another - in a key. // ' " and | are only special if they begin an unquoted key or value. -var UnquotedKeySpecials = string([]rune{'#', ';', '\n', '\\', '{', '}', '[', ']', '\'', '"', '|', ':', '.', '-', '<', '>', '*', '&', '(', ')', '@'}) +var UnquotedKeySpecials = string([]rune{'#', ';', '\n', '\\', '{', '}', '[', ']', '\'', '"', '|', ':', '.', '-', '<', '>', '*', '&', '(', ')', '@', '&'}) var UnquotedValueSpecials = string([]rune{'#', ';', '\n', '\\', '{', '}', '[', ']', '\'', '"', '|', '$', '@'}) // RawString returns s in a AST String node that can format s in the most aesthetically // pleasing way. -// TODO: Return StringBox func RawString(s string, inKey bool) String { if s == "" { return FlatDoubleQuotedString(s) @@ -1064,10 +1135,6 @@ func RawString(s string, inKey bool) String { if i+1 < len(s) && s[i+1] != '-' { continue } - case '&': - if i > 0 { - continue - } } if strings.ContainsRune(UnquotedKeySpecials, r) { if !strings.ContainsRune(s, '"') { diff --git a/d2chaos/d2chaos_test.go b/d2chaos/d2chaos_test.go index 0033cec90..e101d434d 100644 --- a/d2chaos/d2chaos_test.go +++ b/d2chaos/d2chaos_test.go @@ -102,7 +102,7 @@ func test(t *testing.T, textPath, text string) { t.Fatal(err) } - g, err := d2compiler.Compile("", strings.NewReader(text), nil) + g, _, err := d2compiler.Compile("", strings.NewReader(text), nil) if err != nil { t.Fatal(err) } diff --git a/d2cli/main.go b/d2cli/main.go index 3d4fa6054..546a5a977 100644 --- a/d2cli/main.go +++ b/d2cli/main.go @@ -20,6 +20,7 @@ import ( "oss.terrastruct.com/util-go/go2" "oss.terrastruct.com/util-go/xmain" + "oss.terrastruct.com/d2/d2graph" "oss.terrastruct.com/d2/d2lib" "oss.terrastruct.com/d2/d2parser" "oss.terrastruct.com/d2/d2plugin" @@ -66,7 +67,8 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { } debugFlag, err := ms.Opts.Bool("DEBUG", "debug", "d", false, "print debug logs.") if err != nil { - return err + ms.Log.Warn.Printf("Invalid DEBUG flag value ignored") + debugFlag = go2.Pointer(false) } imgCacheFlag, err := ms.Opts.Bool("IMG_CACHE", "img-cache", "", true, "in watch mode, images used in icons are cached for subsequent compilations. This should be disabled if images might change.") if err != nil { @@ -117,11 +119,11 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { fontBoldFlag := ms.Opts.String("D2_FONT_BOLD", "font-bold", "", "", "path to .ttf file to use for the bold font. If none provided, Source Sans Pro Bold is used.") fontSemiboldFlag := ms.Opts.String("D2_FONT_SEMIBOLD", "font-semibold", "", "", "path to .ttf file to use for the semibold font. If none provided, Source Sans Pro Semibold is used.") - ps, err := d2plugin.ListPlugins(ctx) + plugins, err := d2plugin.ListPlugins(ctx) if err != nil { return err } - err = populateLayoutOpts(ctx, ms, ps) + err = populateLayoutOpts(ctx, ms, plugins) if err != nil { return err } @@ -146,7 +148,7 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { case "init-playwright": return initPlaywright() case "layout": - return layoutCmd(ctx, ms, ps) + return layoutCmd(ctx, ms, plugins) case "themes": themesCmd(ctx, ms) return nil @@ -226,6 +228,38 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { } ms.Log.Debug.Printf("using theme %s (ID: %d)", match.Name, *themeFlag) + // If flag is not explicitly set by user, set to nil. + // Later, configs from D2 code will only overwrite if they weren't explicitly set by user + flagSet := make(map[string]struct{}) + ms.Opts.Flags.Visit(func(f *pflag.Flag) { + flagSet[f.Name] = struct{}{} + }) + if ms.Env.Getenv("D2_LAYOUT") == "" { + if _, ok := flagSet["layout"]; !ok { + layoutFlag = nil + } + } + if ms.Env.Getenv("D2_THEME") == "" { + if _, ok := flagSet["theme"]; !ok { + themeFlag = nil + } + } + if ms.Env.Getenv("D2_SKETCH") == "" { + if _, ok := flagSet["sketch"]; !ok { + sketchFlag = nil + } + } + if ms.Env.Getenv("D2_PAD") == "" { + if _, ok := flagSet["pad"]; !ok { + padFlag = nil + } + } + if ms.Env.Getenv("D2_CENTER") == "" { + if _, ok := flagSet["center"]; !ok { + centerFlag = nil + } + } + if *darkThemeFlag == -1 { darkThemeFlag = nil // TODO this is a temporary solution: https://github.com/terrastruct/util-go/issues/7 } @@ -241,29 +275,6 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { scale = scaleFlag } - plugin, err := d2plugin.FindPlugin(ctx, ps, *layoutFlag) - if err != nil { - if errors.Is(err, exec.ErrNotFound) { - return layoutNotFound(ctx, ps, *layoutFlag) - } - return err - } - - err = d2plugin.HydratePluginOpts(ctx, ms, plugin) - if err != nil { - return err - } - - pinfo, err := plugin.Info(ctx) - if err != nil { - return err - } - plocation := pinfo.Type - if pinfo.Type == "binary" { - plocation = fmt.Sprintf("executable plugin at %s", humanPath(pinfo.Path)) - } - ms.Log.Debug.Printf("using layout plugin %s (%s)", *layoutFlag, plocation) - if !outputFormat.supportsDarkTheme() { if darkThemeFlag != nil { ms.Log.Warn.Printf("--dark-theme cannot be used while exporting to another format other than .svg") @@ -285,10 +296,10 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { } renderOpts := d2svg.RenderOpts{ - Pad: int(*padFlag), - Sketch: *sketchFlag, - Center: *centerFlag, - ThemeID: *themeFlag, + Pad: padFlag, + Sketch: sketchFlag, + Center: centerFlag, + ThemeID: themeFlag, DarkThemeID: darkThemeFlag, Scale: scale, } @@ -298,7 +309,8 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { return xmain.UsageErrorf("-w[atch] cannot be combined with reading input from stdin") } w, err := newWatcher(ctx, ms, watcherOpts{ - layoutPlugin: plugin, + plugins: plugins, + layout: layoutFlag, renderOpts: renderOpts, animateInterval: *animateIntervalFlag, host: *hostFlag, @@ -319,7 +331,7 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { ctx, cancel := timelib.WithTimeout(ctx, time.Minute*2) defer cancel() - _, written, err := compile(ctx, ms, plugin, renderOpts, fontFamily, *animateIntervalFlag, inputPath, outputPath, *bundleFlag, *forceAppendixFlag, pw.Page) + _, written, err := compile(ctx, ms, plugins, layoutFlag, renderOpts, fontFamily, *animateIntervalFlag, inputPath, outputPath, "", *bundleFlag, *forceAppendixFlag, pw.Page) if err != nil { if written { return fmt.Errorf("failed to fully compile (partial render written): %w", err) @@ -329,7 +341,32 @@ func Run(ctx context.Context, ms *xmain.State) (err error) { return nil } -func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, renderOpts d2svg.RenderOpts, fontFamily *d2fonts.FontFamily, animateInterval int64, inputPath, outputPath string, bundle, forceAppendix bool, page playwright.Page) (_ []byte, written bool, _ error) { +func LayoutResolver(ctx context.Context, ms *xmain.State, plugins []d2plugin.Plugin) func(engine string) (d2graph.LayoutGraph, error) { + cached := make(map[string]d2graph.LayoutGraph) + return func(engine string) (d2graph.LayoutGraph, error) { + if c, ok := cached[engine]; ok { + return c, nil + } + + plugin, err := d2plugin.FindPlugin(ctx, plugins, engine) + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return nil, layoutNotFound(ctx, plugins, engine) + } + return nil, err + } + + err = d2plugin.HydratePluginOpts(ctx, ms, plugin) + if err != nil { + return nil, err + } + + cached[engine] = plugin.Layout + return plugin.Layout, nil + } +} + +func compile(ctx context.Context, ms *xmain.State, plugins []d2plugin.Plugin, layout *string, renderOpts d2svg.RenderOpts, fontFamily *d2fonts.FontFamily, animateInterval int64, inputPath, outputPath, boardPath string, bundle, forceAppendix bool, page playwright.Page) (_ []byte, written bool, _ error) { start := time.Now() input, err := ms.ReadPath(inputPath) if err != nil { @@ -341,16 +378,12 @@ func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, rende return nil, false, err } - layout := plugin.Layout opts := &d2lib.CompileOptions{ - Layout: layout, - Ruler: ruler, - ThemeID: renderOpts.ThemeID, - FontFamily: fontFamily, - InputPath: inputPath, - } - if renderOpts.Sketch { - opts.FontFamily = go2.Pointer(d2fonts.HandDrawn) + Ruler: ruler, + FontFamily: fontFamily, + InputPath: inputPath, + LayoutResolver: LayoutResolver(ctx, ms, plugins), + Layout: layout, } cancel := background.Repeat(func() { @@ -358,12 +391,14 @@ func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, rende }, time.Second*5) defer cancel() - diagram, g, err := d2lib.Compile(ctx, string(input), opts) + diagram, g, err := d2lib.Compile(ctx, string(input), opts, &renderOpts) if err != nil { return nil, false, err } cancel() + plugin, _ := d2plugin.FindPlugin(ctx, plugins, *opts.Layout) + if animateInterval > 0 { masterID, err := diagram.HashID() if err != nil { @@ -372,6 +407,16 @@ func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, rende renderOpts.MasterID = masterID } + pinfo, err := plugin.Info(ctx) + if err != nil { + return nil, false, err + } + plocation := pinfo.Type + if pinfo.Type == "binary" { + plocation = fmt.Sprintf("executable plugin at %s", humanPath(pinfo.Path)) + } + ms.Log.Debug.Printf("using layout plugin %s (%s)", *opts.Layout, plocation) + pluginInfo, err := plugin.Info(ctx) if err != nil { return nil, false, err @@ -455,7 +500,12 @@ func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, rende } } - boards, err := render(ctx, ms, compileDur, plugin, renderOpts, inputPath, outputPath, bundle, forceAppendix, page, ruler, diagram) + board := diagram.GetBoard(boardPath) + if board == nil { + return nil, false, fmt.Errorf("Diagram with path %s not found", boardPath) + } + + boards, err := render(ctx, ms, compileDur, plugin, renderOpts, inputPath, outputPath, bundle, forceAppendix, page, ruler, board) if err != nil { return nil, false, err } @@ -805,7 +855,7 @@ func renderPDF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opt if err != nil { return svg, err } - err = doc.AddPDFPage(pngImg, boardPath, opts.ThemeID, rootFill, diagram.Shapes, int64(opts.Pad), viewboxX, viewboxY, pageMap) + err = doc.AddPDFPage(pngImg, boardPath, *opts.ThemeID, rootFill, diagram.Shapes, *opts.Pad, viewboxX, viewboxY, pageMap) if err != nil { return svg, err } diff --git a/d2cli/static/watch.js b/d2cli/static/watch.js index a95403971..4d91259eb 100644 --- a/d2cli/static/watch.js +++ b/d2cli/static/watch.js @@ -8,9 +8,7 @@ function init(reconnectDelay) { const d2SVG = window.document.querySelector("#d2-svg-container"); const devMode = document.body.dataset.d2DevMode === "true"; - const ws = new WebSocket( - `ws://${window.location.host}${window.location.pathname}watch` - ); + const ws = new WebSocket(`ws://${window.location.host}/watch`); let isInit = true; let ratio; ws.onopen = () => { @@ -28,7 +26,7 @@ function init(reconnectDelay) { // we can't just set `d2SVG.innerHTML = msg.svg` need to parse this as xml not html const parsedXML = new DOMParser().parseFromString(msg.svg, "text/xml"); d2SVG.replaceChildren(parsedXML.documentElement); - changeFavicon("./static/favicon.ico"); + changeFavicon("/static/favicon.ico"); const svgEl = d2SVG.querySelector("#d2-svg"); // just use inner SVG in watch mode svgEl.parentElement.replaceWith(svgEl); @@ -60,7 +58,7 @@ function init(reconnectDelay) { if (msg.err) { d2ErrDiv.innerText = msg.err; d2ErrDiv.style.display = "block"; - changeFavicon("./static/favicon-err.ico"); + changeFavicon("/static/favicon-err.ico"); d2ErrDiv.scrollIntoView(); } }; diff --git a/d2cli/watch.go b/d2cli/watch.go index bb006eb17..866a480b3 100644 --- a/d2cli/watch.go +++ b/d2cli/watch.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "sync" "time" @@ -41,13 +42,15 @@ var devMode = false var staticFS embed.FS type watcherOpts struct { - layoutPlugin d2plugin.Plugin + layout *string + plugins []d2plugin.Plugin renderOpts d2svg.RenderOpts animateInterval int64 host string port string inputPath string outputPath string + boardPath string pwd string bundle bool forceAppendix bool @@ -361,7 +364,7 @@ func (w *watcher) compileLoop(ctx context.Context) error { w.pw = newPW } - svg, _, err := compile(ctx, w.ms, w.layoutPlugin, w.renderOpts, w.fontFamily, w.animateInterval, w.inputPath, w.outputPath, w.bundle, w.forceAppendix, w.pw.Page) + svg, _, err := compile(ctx, w.ms, w.plugins, w.layout, w.renderOpts, w.fontFamily, w.animateInterval, w.inputPath, w.outputPath, w.boardPath, w.bundle, w.forceAppendix, w.pw.Page) errs := "" if err != nil { if len(svg) > 0 { @@ -429,15 +432,25 @@ func (w *watcher) handleRoot(hw http.ResponseWriter, r *http.Request) {