From 8e76cbd3c88e2f08a2431e8b3080750328147a3a Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 30 Nov 2022 02:34:25 -0800 Subject: [PATCH 01/49] watch.css: Fix from d2c -> d2 rename cc @alixander this has been broken for so long lol From when the code was in the monorepo. --- cmd/d2/static/watch.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/d2/static/watch.css b/cmd/d2/static/watch.css index 8c5bccc9a..3e389ddbb 100644 --- a/cmd/d2/static/watch.css +++ b/cmd/d2/static/watch.css @@ -1,4 +1,4 @@ -#d2c-err { +#d2-err { /* This style was copied from Chrome's svg parser error style. */ white-space: pre-wrap; border: 2px solid #c77; From f8418f3a2c17bf076b4178849982f8fa639f3b5d Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 30 Nov 2022 02:35:42 -0800 Subject: [PATCH 02/49] xmain: Return usage errors automatically from opts --- cmd/d2/main.go | 10 +++++----- lib/xmain/opts.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/d2/main.go b/cmd/d2/main.go index 063783e37..d4523ae43 100644 --- a/cmd/d2/main.go +++ b/cmd/d2/main.go @@ -37,26 +37,26 @@ func run(ctx context.Context, ms *xmain.State) (err error) { // These should be kept up-to-date with the d2 man page watchFlag, err := ms.Opts.Bool("D2_WATCH", "watch", "w", false, "watch for changes to input and live reload. Use $HOST and $PORT to specify the listening address.\n(default localhost:0, which is will open on a randomly available local port).") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } hostFlag := ms.Opts.String("HOST", "host", "h", "localhost", "host listening address when used with watch") portFlag := ms.Opts.String("PORT", "port", "p", "0", "port listening address when used with watch") bundleFlag, err := ms.Opts.Bool("D2_BUNDLE", "bundle", "b", true, "when outputting SVG, bundle all assets and layers into the output file.") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } debugFlag, err := ms.Opts.Bool("DEBUG", "debug", "d", false, "print debug logs.") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } layoutFlag := ms.Opts.String("D2_LAYOUT", "layout", "l", "dagre", `the layout engine used.`) themeFlag, err := ms.Opts.Int64("D2_THEME", "theme", "t", 0, "the diagram theme ID. For a list of available options, see https://oss.terrastruct.com/d2") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } versionFlag, err := ms.Opts.Bool("", "version", "v", false, "get the version") if err != nil { - return xmain.UsageErrorf(err.Error()) + return err } err = ms.Opts.Flags.Parse(ms.Opts.Args) diff --git a/lib/xmain/opts.go b/lib/xmain/opts.go index 98d9387ef..22a96b0ab 100644 --- a/lib/xmain/opts.go +++ b/lib/xmain/opts.go @@ -129,7 +129,7 @@ func (o *Opts) Int64(envKey, flag, shortFlag string, defaultVal int64, usage str if env := o.getEnv(flag, envKey); env != "" { envVal, err := strconv.ParseInt(env, 10, 64) if err != nil { - return nil, fmt.Errorf(`invalid environment variable %s. Expected int64. Found "%v".`, envKey, envVal) + return nil, UsageErrorf(`invalid environment variable %s. Expected int64. Found "%v".`, envKey, envVal) } defaultVal = envVal } @@ -148,7 +148,7 @@ func (o *Opts) String(envKey, flag, shortFlag string, defaultVal, usage string) func (o *Opts) Bool(envKey, flag, shortFlag string, defaultVal bool, usage string) (*bool, error) { if env := o.getEnv(flag, envKey); env != "" { if !boolyEnv(env) { - return nil, fmt.Errorf(`invalid environment variable %s. Expected bool. Found "%s".`, envKey, env) + return nil, UsageErrorf(`invalid environment variable %s. Expected bool. Found "%s".`, envKey, env) } if truthyEnv(env) { defaultVal = true From 0edf30a6cd158438ce1604eea052e39fe8c5702d Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 30 Nov 2022 02:36:29 -0800 Subject: [PATCH 03/49] imgbundler: Fixes - Make bundle flag work - Display error and update render at the same time in watch mode. Before we would just display the render and not show the error. - Rename imgbundler.InlineX functions to BundleX - Print imgbundler fetch/readFile errors as they happen in the workers instead of coalescing and printing at the end. - Minor performance improvements by using []byte everywhere possible. - Improved symbol naming in imgbundler code - **major**: Ignore already bundled images instead of trying to os.ReadFile them. --- cmd/d2/main.go | 43 ++++----- cmd/d2/static/watch.js | 16 ++-- cmd/d2/watch.go | 25 +++-- go.mod | 2 +- lib/imgbundler/imgbundler.go | 154 +++++++++++++++--------------- lib/imgbundler/imgbundler_test.go | 20 ++-- 6 files changed, 133 insertions(+), 127 deletions(-) diff --git a/cmd/d2/main.go b/cmd/d2/main.go index d4523ae43..7db63372d 100644 --- a/cmd/d2/main.go +++ b/cmd/d2/main.go @@ -12,6 +12,7 @@ import ( "github.com/playwright-community/playwright-go" "github.com/spf13/pflag" + "go.uber.org/multierr" "oss.terrastruct.com/d2" "oss.terrastruct.com/d2/d2layouts/d2sequence" @@ -156,6 +157,7 @@ func run(ctx context.Context, ms *xmain.State) (err error) { port: *portFlag, inputPath: inputPath, outputPath: outputPath, + bundle: *bundleFlag, pw: pw, }) if err != nil { @@ -167,19 +169,15 @@ func run(ctx context.Context, ms *xmain.State) (err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute*2) defer cancel() - if *bundleFlag { - _ = 343 - } - - _, err = compile(ctx, ms, false, plugin, *themeFlag, inputPath, outputPath, pw.Page) + _, err = compile(ctx, ms, plugin, *themeFlag, inputPath, outputPath, *bundleFlag, pw.Page) if err != nil { - return err + return fmt.Errorf("failed to compile: %w", err) } ms.Log.Success.Printf("successfully compiled %v to %v", inputPath, outputPath) return nil } -func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, page playwright.Page) ([]byte, error) { +func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, bundle bool, page playwright.Page) ([]byte, error) { input, err := ms.ReadPath(inputPath) if err != nil { return nil, err @@ -210,38 +208,37 @@ func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plu } svg, err = plugin.PostProcess(ctx, svg) if err != nil { - return nil, err + return svg, err } - svg, err = imgbundler.InlineLocal(ctx, ms, svg) - if err != nil { - ms.Log.Error.Printf("missing/broken local image(s), writing partial output: %v", err) + + svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg) + if bundle { + var bundleErr2 error + svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg) + bundleErr = multierr.Combine(bundleErr, bundleErr2) } out := svg if filepath.Ext(outputPath) == ".png" { - svg, err = imgbundler.InlineRemote(ctx, ms, svg) - if err != nil { - ms.Log.Error.Printf("missing/broken remote image(s), writing partial output: %v", err) + svg := svg + if !bundle { + var bundleErr2 error + svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg) + bundleErr = multierr.Combine(bundleErr, bundleErr2) } out, err = png.ConvertSVG(ms, page, svg) if err != nil { - return nil, err + return svg, err } } err = ms.WritePath(outputPath, out) if err != nil { - return nil, err + return svg, err } - // Missing/broken images are fine during watch mode, as the user is likely building up a diagram. - // Otherwise, the assumption is that this diagram is building for production, and broken images are not okay. - if !isWatching && ms.Log.Nerrors() > 0 { - return nil, xmain.ExitErrorf(1, "errors logged while rendering, partial output written to %v", outputPath) - } - - return svg, nil + return svg, bundleErr } // newExt must include leading . diff --git a/cmd/d2/static/watch.js b/cmd/d2/static/watch.js index e2c9adaf3..868fb741b 100644 --- a/cmd/d2/static/watch.js +++ b/cmd/d2/static/watch.js @@ -4,6 +4,9 @@ window.addEventListener("DOMContentLoaded", () => { }); function init(reconnectDelay) { + const d2ErrDiv = window.document.querySelector("#d2-err"); + const d2SVG = window.document.querySelector("#d2-svg"); + const devMode = document.body.dataset.d2DevMode === "true"; const ws = new WebSocket( `ws://${window.location.host}${window.location.pathname}watch` @@ -19,13 +22,7 @@ function init(reconnectDelay) { } else { console.debug("watch websocket received data"); } - const d2ErrDiv = window.document.querySelector("#d2-err"); - if (msg.err) { - d2ErrDiv.innerText = msg.err; - d2ErrDiv.style.display = "block"; - d2ErrDiv.scrollIntoView(); - } else { - const d2SVG = window.document.querySelector("#d2-svg"); + if (msg.svg) { // We could turn d2SVG into an actual SVG element and use outerHTML to fully replace it // with the result from the renderer but unfortunately that overwrites the #d2-svg ID. // Even if you add another line to set it afterwards. The parsing/interpretation of outerHTML must be async. @@ -36,6 +33,11 @@ function init(reconnectDelay) { d2SVG.innerHTML = msg.svg; d2ErrDiv.style.display = "none"; } + if (msg.err) { + d2ErrDiv.innerText = msg.err; + d2ErrDiv.style.display = "block"; + d2ErrDiv.scrollIntoView(); + } }; ws.onerror = (ev) => { console.error("watch websocket connection error", ev); diff --git a/cmd/d2/watch.go b/cmd/d2/watch.go index ed528ead3..4cc5ed81f 100644 --- a/cmd/d2/watch.go +++ b/cmd/d2/watch.go @@ -42,6 +42,7 @@ type watcherOpts struct { port string inputPath string outputPath string + bundle bool pw png.Playwright } @@ -73,8 +74,8 @@ type watcher struct { } type compileResult struct { - Err string `json:"err"` SVG string `json:"svg"` + Err string `json:"err"` } func newWatcher(ctx context.Context, ms *xmain.State, opts watcherOpts) (*watcher, error) { @@ -345,19 +346,23 @@ func (w *watcher) compileLoop(ctx context.Context) error { w.pw = newPW } - b, err := compile(ctx, w.ms, true, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.pw.Page) + b, err := compile(ctx, w.ms, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.bundle, w.pw.Page) + errs := "" if err != nil { - err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err) - w.ms.Log.Error.Print(err) - w.broadcast(&compileResult{ - Err: err.Error(), - }) + if len(b) > 0 { + err = fmt.Errorf("failed to %scompile (rendering partial output): %w", recompiledPrefix, err) + } else { + err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err) + } + errs = err.Error() + w.ms.Log.Error.Print(errs) } else { w.ms.Log.Success.Printf("successfully %scompiled %v to %v", recompiledPrefix, w.inputPath, w.outputPath) - w.broadcast(&compileResult{ - SVG: string(b), - }) } + w.broadcast(&compileResult{ + SVG: string(b), + Err: errs, + }) if firstCompile { firstCompile = false diff --git a/go.mod b/go.mod index fc891de0f..2b8bb90d2 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( golang.org/x/image v0.1.0 golang.org/x/net v0.2.0 golang.org/x/text v0.4.0 + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 gonum.org/v1/plot v0.12.0 nhooyr.io/websocket v1.8.7 oss.terrastruct.com/cmdlog v0.0.0-20221129200109-540ef52ff07d @@ -57,7 +58,6 @@ require ( golang.org/x/crypto v0.3.0 // indirect golang.org/x/sys v0.2.0 // indirect golang.org/x/term v0.2.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/lib/imgbundler/imgbundler.go b/lib/imgbundler/imgbundler.go index 82e7105b9..207809c2e 100644 --- a/lib/imgbundler/imgbundler.go +++ b/lib/imgbundler/imgbundler.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "go.uber.org/multierr" + "golang.org/x/xerrors" "oss.terrastruct.com/xdefer" "oss.terrastruct.com/d2/lib/xmain" @@ -22,140 +22,142 @@ import ( const maxImageSize int64 = 1 << 25 // 33_554_432 -var imageRe = regexp.MustCompile(` 0 { + return svg, xerrors.Errorf("failed to bundle the following images: %v", errhrefs) + } + return svg, nil } - if resp.err != nil { - err = multierr.Combine(err, resp.err) - continue - } - svg = bytes.Replace(svg, []byte(resp.srctxt), []byte(fmt.Sprintf(``, href) - matches := imageRe.FindAllStringSubmatch(str, -1) + matches := imageRegex.FindAllStringSubmatch(str, -1) if len(matches) != 1 { t.Fatalf("uri regex didn't match %s", str) } @@ -90,7 +90,7 @@ width="328" height="587" viewBox="-100 -131 328 587">