diff --git a/ci/release/changelogs/next.md b/ci/release/changelogs/next.md index 9705e0993..cf91ce05e 100644 --- a/ci/release/changelogs/next.md +++ b/ci/release/changelogs/next.md @@ -12,6 +12,8 @@ - Querying shapes and connections by ID is now supported in renders. [#218](https://github.com/terrastruct/d2/pull/218) - [install.sh](./install.sh) now accepts `-d` as an alias for `--dry-run`. [#266](https://github.com/terrastruct/d2/pull/266) +- `-b/--bundle` flag to `d2` now works and bundles all image assets directly as base64 + data urls. [#278](https://github.com/terrastruct/d2/pull/278) #### Improvements 🔧 diff --git a/ci/release/template/man/d2.1 b/ci/release/template/man/d2.1 index 8cf1ce5e2..d46ca20bf 100644 --- a/ci/release/template/man/d2.1 +++ b/ci/release/template/man/d2.1 @@ -9,14 +9,7 @@ .Op Fl -watch Ar false .Op Fl -theme Em 0 .Ar file.d2 -.Op Ar file.svg -| -.Op Ar file.png -.Nm d2 -.Op Fl -watch Ar false -.Op Fl -theme Em 0 -.Ar file.d2 -.Op Ar ... +.Op Ar file.svg | file.png .Nm d2 .Ar layout Op Ar name .Sh DESCRIPTION @@ -29,10 +22,21 @@ to .Ar file.png .Ns . .Pp +It defaults to +.Ar file.svg +if no output path is passed. +.Pp Pass - to have .Nm read from stdin or write to stdout. .Pp +Never use the presence of the output file to check for success. +Always use the exit status of +.Nm d2 +.Ns . +This is because sometimes when errors occur while rendering, d2 still write out a partial +render anyway to enable iteration on a broken diagram. +.Pp See more docs, the source code and license at .Lk https://oss.terrastruct.com/d2 .Sh OPTIONS diff --git a/cmd/d2/help.go b/cmd/d2/help.go index 98390495d..947c601e8 100644 --- a/cmd/d2/help.go +++ b/cmd/d2/help.go @@ -15,11 +15,15 @@ import ( func help(ms *xmain.State) { fmt.Fprintf(ms.Stdout, `Usage: - %s [--watch=false] [--theme=0] file.d2 [file.svg|file.png] + %s [--watch=false] [--theme=0] file.d2 [file.svg | file.png] + +%[1]s compiles and renders file.d2 to file.svg | file.png +It defaults to file.svg if an output path is not provided. -%[1]s compiles and renders file.d2 to file.svg|file.png. Use - to have d2 read from stdin or write to stdout. +See man %[1]s for more detailed docs. + Flags: %s diff --git a/cmd/d2/main.go b/cmd/d2/main.go index 063783e37..ffcb4509d 100644 --- a/cmd/d2/main.go +++ b/cmd/d2/main.go @@ -12,6 +12,7 @@ import ( "github.com/playwright-community/playwright-go" "github.com/spf13/pflag" + "go.uber.org/multierr" "oss.terrastruct.com/d2" "oss.terrastruct.com/d2/d2layouts/d2sequence" @@ -37,26 +38,26 @@ func run(ctx context.Context, ms *xmain.State) (err error) { // These should be kept up-to-date with the d2 man page watchFlag, err := ms.Opts.Bool("D2_WATCH", "watch", "w", false, "watch for changes to input and live reload. Use $HOST and $PORT to specify the listening address.\n(default localhost:0, which is will open on a randomly available local port).") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } hostFlag := ms.Opts.String("HOST", "host", "h", "localhost", "host listening address when used with watch") portFlag := ms.Opts.String("PORT", "port", "p", "0", "port listening address when used with watch") bundleFlag, err := ms.Opts.Bool("D2_BUNDLE", "bundle", "b", true, "when outputting SVG, bundle all assets and layers into the output file.") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } debugFlag, err := ms.Opts.Bool("DEBUG", "debug", "d", false, "print debug logs.") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } layoutFlag := ms.Opts.String("D2_LAYOUT", "layout", "l", "dagre", `the layout engine used.`) themeFlag, err := ms.Opts.Int64("D2_THEME", "theme", "t", 0, "the diagram theme ID. For a list of available options, see https://oss.terrastruct.com/d2") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } versionFlag, err := ms.Opts.Bool("", "version", "v", false, "get the version") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } err = ms.Opts.Flags.Parse(ms.Opts.Args) @@ -156,6 +157,7 @@ func run(ctx context.Context, ms *xmain.State) (err error) { port: *portFlag, inputPath: inputPath, outputPath: outputPath, + bundle: *bundleFlag, pw: pw, }) if err != nil { @@ -167,27 +169,26 @@ func run(ctx context.Context, ms *xmain.State) (err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute*2) defer cancel() - if *bundleFlag { - _ = 343 - } - - _, err = compile(ctx, ms, false, plugin, *themeFlag, inputPath, outputPath, pw.Page) + _, written, err := compile(ctx, ms, plugin, *themeFlag, inputPath, outputPath, *bundleFlag, pw.Page) if err != nil { - return err + if written { + return fmt.Errorf("failed to fully compile (partial render written): %w", err) + } + return fmt.Errorf("failed to compile: %w", err) } ms.Log.Success.Printf("successfully compiled %v to %v", inputPath, outputPath) return nil } -func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, page playwright.Page) ([]byte, error) { +func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, bundle bool, page playwright.Page) (_ []byte, written bool, _ error) { input, err := ms.ReadPath(inputPath) if err != nil { - return nil, err + return nil, false, err } ruler, err := textmeasure.NewRuler() if err != nil { - return nil, err + return nil, false, err } layout := plugin.Layout @@ -201,47 +202,46 @@ func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plu ThemeID: themeID, }) if err != nil { - return nil, err + return nil, false, err } svg, err := d2svg.Render(d) if err != nil { - return nil, err + return nil, false, err } svg, err = plugin.PostProcess(ctx, svg) if err != nil { - return nil, err + return svg, false, err } - svg, err = imgbundler.InlineLocal(ctx, ms, svg) - if err != nil { - ms.Log.Error.Printf("missing/broken local image(s), writing partial output: %v", err) + + svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg) + if bundle { + var bundleErr2 error + svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg) + bundleErr = multierr.Combine(bundleErr, bundleErr2) } out := svg if filepath.Ext(outputPath) == ".png" { - svg, err = imgbundler.InlineRemote(ctx, ms, svg) - if err != nil { - ms.Log.Error.Printf("missing/broken remote image(s), writing partial output: %v", err) + svg := svg + if !bundle { + var bundleErr2 error + svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg) + bundleErr = multierr.Combine(bundleErr, bundleErr2) } out, err = png.ConvertSVG(ms, page, svg) if err != nil { - return nil, err + return svg, false, err } } err = ms.WritePath(outputPath, out) if err != nil { - return nil, err + return svg, false, err } - // Missing/broken images are fine during watch mode, as the user is likely building up a diagram. - // Otherwise, the assumption is that this diagram is building for production, and broken images are not okay. - if !isWatching && ms.Log.Nerrors() > 0 { - return nil, xmain.ExitErrorf(1, "errors logged while rendering, partial output written to %v", outputPath) - } - - return svg, nil + return svg, true, bundleErr } // newExt must include leading . diff --git a/cmd/d2/static/watch.css b/cmd/d2/static/watch.css index 8c5bccc9a..3e389ddbb 100644 --- a/cmd/d2/static/watch.css +++ b/cmd/d2/static/watch.css @@ -1,4 +1,4 @@ -#d2c-err { +#d2-err { /* This style was copied from Chrome's svg parser error style. */ white-space: pre-wrap; border: 2px solid #c77; diff --git a/cmd/d2/static/watch.js b/cmd/d2/static/watch.js index e2c9adaf3..868fb741b 100644 --- a/cmd/d2/static/watch.js +++ b/cmd/d2/static/watch.js @@ -4,6 +4,9 @@ window.addEventListener("DOMContentLoaded", () => { }); function init(reconnectDelay) { + const d2ErrDiv = window.document.querySelector("#d2-err"); + const d2SVG = window.document.querySelector("#d2-svg"); + const devMode = document.body.dataset.d2DevMode === "true"; const ws = new WebSocket( `ws://${window.location.host}${window.location.pathname}watch` @@ -19,13 +22,7 @@ function init(reconnectDelay) { } else { console.debug("watch websocket received data"); } - const d2ErrDiv = window.document.querySelector("#d2-err"); - if (msg.err) { - d2ErrDiv.innerText = msg.err; - d2ErrDiv.style.display = "block"; - d2ErrDiv.scrollIntoView(); - } else { - const d2SVG = window.document.querySelector("#d2-svg"); + if (msg.svg) { // We could turn d2SVG into an actual SVG element and use outerHTML to fully replace it // with the result from the renderer but unfortunately that overwrites the #d2-svg ID. // Even if you add another line to set it afterwards. The parsing/interpretation of outerHTML must be async. @@ -36,6 +33,11 @@ function init(reconnectDelay) { d2SVG.innerHTML = msg.svg; d2ErrDiv.style.display = "none"; } + if (msg.err) { + d2ErrDiv.innerText = msg.err; + d2ErrDiv.style.display = "block"; + d2ErrDiv.scrollIntoView(); + } }; ws.onerror = (ev) => { console.error("watch websocket connection error", ev); diff --git a/cmd/d2/watch.go b/cmd/d2/watch.go index ed528ead3..a09034e4d 100644 --- a/cmd/d2/watch.go +++ b/cmd/d2/watch.go @@ -42,6 +42,7 @@ type watcherOpts struct { port string inputPath string outputPath string + bundle bool pw png.Playwright } @@ -73,8 +74,8 @@ type watcher struct { } type compileResult struct { - Err string `json:"err"` SVG string `json:"svg"` + Err string `json:"err"` } func newWatcher(ctx context.Context, ms *xmain.State, opts watcherOpts) (*watcher, error) { @@ -345,19 +346,23 @@ func (w *watcher) compileLoop(ctx context.Context) error { w.pw = newPW } - b, err := compile(ctx, w.ms, true, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.pw.Page) + svg, _, err := compile(ctx, w.ms, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.bundle, w.pw.Page) + errs := "" if err != nil { - err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err) - w.ms.Log.Error.Print(err) - w.broadcast(&compileResult{ - Err: err.Error(), - }) + if len(svg) > 0 { + err = fmt.Errorf("failed to fully %scompile (rendering partial svg): %w", recompiledPrefix, err) + } else { + err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err) + } + errs = err.Error() + w.ms.Log.Error.Print(errs) } else { w.ms.Log.Success.Printf("successfully %scompiled %v to %v", recompiledPrefix, w.inputPath, w.outputPath) - w.broadcast(&compileResult{ - SVG: string(b), - }) } + w.broadcast(&compileResult{ + SVG: string(svg), + Err: errs, + }) if firstCompile { firstCompile = false diff --git a/go.mod b/go.mod index fc891de0f..2b8bb90d2 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( golang.org/x/image v0.1.0 golang.org/x/net v0.2.0 golang.org/x/text v0.4.0 + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 gonum.org/v1/plot v0.12.0 nhooyr.io/websocket v1.8.7 oss.terrastruct.com/cmdlog v0.0.0-20221129200109-540ef52ff07d @@ -57,7 +58,6 @@ require ( golang.org/x/crypto v0.3.0 // indirect golang.org/x/sys v0.2.0 // indirect golang.org/x/term v0.2.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/lib/imgbundler/imgbundler.go b/lib/imgbundler/imgbundler.go index 82e7105b9..0935f8637 100644 --- a/lib/imgbundler/imgbundler.go +++ b/lib/imgbundler/imgbundler.go @@ -6,15 +6,17 @@ import ( "encoding/base64" "fmt" "io/ioutil" + "mime" "net/http" "net/url" "os" + "path" "regexp" "strings" "sync" "time" - "go.uber.org/multierr" + "golang.org/x/xerrors" "oss.terrastruct.com/xdefer" "oss.terrastruct.com/d2/lib/xmain" @@ -22,140 +24,186 @@ import ( const maxImageSize int64 = 1 << 25 // 33_554_432 -var imageRe = regexp.MustCompile(` 0 { + return svg, xerrors.Errorf("%v", errhrefs) + } + return svg, nil } - if resp.err != nil { - err = multierr.Combine(err, resp.err) - continue - } - svg = bytes.Replace(svg, []byte(resp.srctxt), []byte(fmt.Sprintf(``, href) - matches := imageRe.FindAllStringSubmatch(str, -1) + matches := imageRegex.FindAllStringSubmatch(str, -1) if len(matches) != 1 { t.Fatalf("uri regex didn't match %s", str) } @@ -90,7 +90,7 @@ width="328" height="587" viewBox="-100 -131 328 587">