From eba687fbbecfd641e00bf544db1c2942667cb3b2 Mon Sep 17 00:00:00 2001 From: Gavin Nishizawa Date: Thu, 19 Oct 2023 16:06:11 -0700 Subject: [PATCH 1/5] don't use xmain.State in imgbundler library --- d2cli/main.go | 22 ++++++++++++--------- lib/imgbundler/imgbundler.go | 32 +++++++++++++++---------------- lib/imgbundler/imgbundler_test.go | 22 ++++++++++----------- 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/d2cli/main.go b/d2cli/main.go index 56aed09e9..7963f9693 100644 --- a/d2cli/main.go +++ b/d2cli/main.go @@ -748,10 +748,11 @@ func _render(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opts return svg, err } - svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg) + cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" + svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) if bundle { var bundleErr2 error - svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg) + svg, bundleErr2 = imgbundler.BundleRemote(ctx, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) } if forceAppendix && !toPNG { @@ -764,7 +765,7 @@ func _render(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opts if !bundle { var bundleErr2 error - svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg) + svg, bundleErr2 = imgbundler.BundleRemote(ctx, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) } @@ -833,8 +834,9 @@ func renderPDF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opt return svg, err } - svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg) - svg, bundleErr2 := imgbundler.BundleRemote(ctx, ms, svg) + cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" + svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) + svg, bundleErr2 := imgbundler.BundleRemote(ctx, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) if bundleErr != nil { return svg, bundleErr @@ -933,8 +935,9 @@ func renderPPTX(ctx context.Context, ms *xmain.State, presentation *pptx.Present return nil, err } - svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg) - svg, bundleErr2 := imgbundler.BundleRemote(ctx, ms, svg) + cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" + svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) + svg, bundleErr2 := imgbundler.BundleRemote(ctx, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) if bundleErr != nil { return nil, bundleErr @@ -1178,8 +1181,9 @@ func renderPNGsForGIF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plug return nil, nil, err } - svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg) - svg, bundleErr2 := imgbundler.BundleRemote(ctx, ms, svg) + cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" + svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) + svg, bundleErr2 := imgbundler.BundleRemote(ctx, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) if bundleErr != nil { return nil, nil, bundleErr diff --git a/lib/imgbundler/imgbundler.go b/lib/imgbundler/imgbundler.go index 526f3bf8d..8324ce205 100644 --- a/lib/imgbundler/imgbundler.go +++ b/lib/imgbundler/imgbundler.go @@ -19,8 +19,8 @@ import ( "golang.org/x/xerrors" + "oss.terrastruct.com/d2/lib/log" "oss.terrastruct.com/util-go/xdefer" - "oss.terrastruct.com/util-go/xmain" ) var imgCache sync.Map @@ -29,12 +29,12 @@ const maxImageSize int64 = 1 << 25 // 33_554_432 var imageRegex = regexp.MustCompile(` 0 { @@ -150,8 +150,8 @@ func runWorkers(ctx context.Context, ms *xmain.State, svg []byte, imgs [][][]byt } } -func worker(ctx context.Context, ms *xmain.State, href []byte, isRemote bool) ([]byte, error) { - if ms.Env.Getenv("IMG_CACHE") == "1" { +func worker(ctx context.Context, href []byte, isRemote, cacheImages bool) ([]byte, error) { + if cacheImages { if hit, ok := imgCache.Load(string(href)); ok { return hit.([]byte), nil } @@ -160,10 +160,10 @@ func worker(ctx context.Context, ms *xmain.State, href []byte, isRemote bool) ([ var mimeType string var err error if isRemote { - ms.Log.Debug.Printf("fetching %s remotely", string(href)) + log.Debug(ctx, fmt.Sprintf("fetching %s remotely", string(href))) buf, mimeType, err = httpGet(ctx, html.UnescapeString(string(href))) } else { - ms.Log.Debug.Printf("reading %s from disk", string(href)) + log.Debug(ctx, fmt.Sprintf("reading %s from disk", string(href))) buf, err = os.ReadFile(html.UnescapeString(string(href))) } if err != nil { @@ -177,7 +177,7 @@ func worker(ctx context.Context, ms *xmain.State, href []byte, isRemote bool) ([ b64 := base64.StdEncoding.EncodeToString(buf) out := []byte(fmt.Sprintf(` `, svgURL, pngURL) - ms := &xmain.State{ - Name: "test", - - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - - Env: xos.NewEnv(os.Environ()), - } - ms.Log = cmdlog.NewTB(ms.Env, t) - httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response { respRecorder := httptest.NewRecorder() switch req.URL.String() { @@ -168,7 +153,7 @@ width="328" height="587" viewBox="-100 -131 328 587"> `, svgURL, pngURL) - ms := &xmain.State{ - Name: "test", - - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - - Env: xos.NewEnv(os.Environ()), - } - ms.Log = cmdlog.NewTB(ms.Env, t) out, err := BundleLocal(ctx, []byte(sampleSVG), false) if err != nil { t.Fatal(err) @@ -233,7 +208,7 @@ width="328" height="587" viewBox="-100 -131 328 587"> `, url1, url2) - ms := &xmain.State{ - Name: "test", - - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - - Env: xos.NewEnv(os.Environ()), - } - ms.Log = cmdlog.NewTB(ms.Env, t) - count := 0 httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response { @@ -298,7 +262,7 @@ width="328" height="587" viewBox="-100 -131 328 587"> `, url1, url2) - ms := &xmain.State{ - Name: "test", - - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - - Env: xos.NewEnv(os.Environ()), - } - ms.Log = cmdlog.NewTB(ms.Env, t) - count := 0 httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response { From 4dda484b50d5faf7b90c17169a657409c44c44bd Mon Sep 17 00:00:00 2001 From: Gavin Nishizawa Date: Thu, 19 Oct 2023 16:31:46 -0700 Subject: [PATCH 3/5] test logging --- lib/imgbundler/imgbundler_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/imgbundler/imgbundler_test.go b/lib/imgbundler/imgbundler_test.go index ff1a815df..ae381636d 100644 --- a/lib/imgbundler/imgbundler_test.go +++ b/lib/imgbundler/imgbundler_test.go @@ -12,6 +12,7 @@ import ( "sync" "testing" + "cdr.dev/slog/sloggers/slogtest" tassert "github.com/stretchr/testify/assert" "oss.terrastruct.com/d2/lib/log" @@ -49,7 +50,8 @@ func TestRegex(t *testing.T) { func TestInlineRemote(t *testing.T) { imgCache = sync.Map{} - ctx := log.WithTB(context.Background(), t, nil) + // we don't want log.Error to cause this test to fail + ctx := log.WithTB(context.Background(), t, &slogtest.Options{IgnoreErrors: true}) svgURL := "https://icons.terrastruct.com/essentials/004-picture.svg" pngURL := "https://cdn4.iconfinder.com/data/icons/smart-phones-technologies/512/android-phone.png" From 504eb2fa69be9fe872190dade6e2bc0d6a97081c Mon Sep 17 00:00:00 2001 From: Gavin Nishizawa Date: Thu, 19 Oct 2023 16:54:10 -0700 Subject: [PATCH 4/5] also remove dependency from png and xgif libs --- d2cli/main.go | 28 +++++++++++++++++++++++----- lib/png/png.go | 10 +--------- lib/xgif/xgif.go | 12 +----------- lib/xgif/xgif_test.go | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/d2cli/main.go b/d2cli/main.go index 7963f9693..68ea60b42 100644 --- a/d2cli/main.go +++ b/d2cli/main.go @@ -434,7 +434,7 @@ func compile(ctx context.Context, ms *xmain.State, plugins []d2plugin.Plugin, la if err != nil { return nil, false, err } - out, err := xgif.AnimatePNGs(ms, pngs, int(animateInterval)) + out, err := AnimatePNGs(ms, pngs, int(animateInterval)) if err != nil { return nil, false, err } @@ -769,7 +769,7 @@ func _render(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opts bundleErr = multierr.Combine(bundleErr, bundleErr2) } - out, err = png.ConvertSVG(ms, page, svg) + out, err = ConvertSVG(ms, page, svg) if err != nil { return svg, err } @@ -843,7 +843,7 @@ func renderPDF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opt } svg = appendix.Append(diagram, ruler, svg) - pngImg, err := png.ConvertSVG(ms, page, svg) + pngImg, err := ConvertSVG(ms, page, svg) if err != nil { return svg, err } @@ -945,7 +945,7 @@ func renderPPTX(ctx context.Context, ms *xmain.State, presentation *pptx.Present svg = appendix.Append(diagram, ruler, svg) - pngImg, err := png.ConvertSVG(ms, page, svg) + pngImg, err := ConvertSVG(ms, page, svg) if err != nil { return nil, err } @@ -1191,7 +1191,7 @@ func renderPNGsForGIF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plug svg = appendix.Append(diagram, ruler, svg) - pngImg, err := png.ConvertSVG(ms, page, svg) + pngImg, err := ConvertSVG(ms, page, svg) if err != nil { return nil, nil, err } @@ -1222,3 +1222,21 @@ func renderPNGsForGIF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plug return svg, pngs, nil } + +func ConvertSVG(ms *xmain.State, page playwright.Page, svg []byte) ([]byte, error) { + cancel := background.Repeat(func() { + ms.Log.Info.Printf("converting to PNG...") + }, time.Second*5) + defer cancel() + + return png.ConvertSVG(page, svg) +} + +func AnimatePNGs(ms *xmain.State, pngs [][]byte, animIntervalMs int) ([]byte, error) { + cancel := background.Repeat(func() { + ms.Log.Info.Printf("generating GIF...") + }, time.Second*5) + defer cancel() + + return xgif.AnimatePNGs(pngs, animIntervalMs) +} diff --git a/lib/png/png.go b/lib/png/png.go index c0cc23687..ac21e0e7b 100644 --- a/lib/png/png.go +++ b/lib/png/png.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "fmt" "strings" - "time" _ "embed" @@ -14,9 +13,7 @@ import ( pngstruct "github.com/dsoprea/go-png-image-structure/v2" "github.com/playwright-community/playwright-go" - "oss.terrastruct.com/d2/lib/background" "oss.terrastruct.com/d2/lib/version" - "oss.terrastruct.com/util-go/xmain" ) // ConvertSVG scales the image by 2x @@ -88,12 +85,7 @@ const pngPrefix = "data:image/png;base64," // ConvertSVG converts the given SVG into a PNG. // Note that the resulting PNG has 2x the size (width and height) of the original SVG (see generate_png.js) -func ConvertSVG(ms *xmain.State, page playwright.Page, svg []byte) ([]byte, error) { - cancel := background.Repeat(func() { - ms.Log.Info.Printf("converting to PNG...") - }, time.Second*5) - defer cancel() - +func ConvertSVG(page playwright.Page, svg []byte) ([]byte, error) { encodedSVG := base64.StdEncoding.EncodeToString(svg) pngInterface, err := page.Evaluate(genPNGScript, map[string]interface{}{ "imgString": "data:image/svg+xml;charset=utf-8;base64," + encodedSVG, diff --git a/lib/xgif/xgif.go b/lib/xgif/xgif.go index e70f856c7..d39b9ca15 100644 --- a/lib/xgif/xgif.go +++ b/lib/xgif/xgif.go @@ -15,27 +15,17 @@ import ( "image/color" "image/gif" "image/png" - "time" "github.com/ericpauley/go-quantize/quantize" - "oss.terrastruct.com/d2/lib/background" "oss.terrastruct.com/util-go/go2" - "oss.terrastruct.com/util-go/xmain" ) const INFINITE_LOOP = 0 var BG_COLOR = color.White -func AnimatePNGs(ms *xmain.State, pngs [][]byte, animIntervalMs int) ([]byte, error) { - if ms != nil { - cancel := background.Repeat(func() { - ms.Log.Info.Printf("generating GIF...") - }, time.Second*5) - defer cancel() - } - +func AnimatePNGs(pngs [][]byte, animIntervalMs int) ([]byte, error) { var width, height int pngImgs := make([]image.Image, len(pngs)) for i, pngBytes := range pngs { diff --git a/lib/xgif/xgif_test.go b/lib/xgif/xgif_test.go index bc3cc1061..fcdc370e7 100644 --- a/lib/xgif/xgif_test.go +++ b/lib/xgif/xgif_test.go @@ -20,7 +20,7 @@ var test_output []byte func TestPngToGif(t *testing.T) { boards := [][]byte{test_input1, test_input2} interval := 1_000 - gifBytes, err := AnimatePNGs(nil, boards, interval) + gifBytes, err := AnimatePNGs(boards, interval) assert.NoError(t, err) // use this to update the test output From 32fbe7e4a8d25762f9d74dbf302ddd2bd2798f00 Mon Sep 17 00:00:00 2001 From: Gavin Nishizawa Date: Thu, 19 Oct 2023 17:44:39 -0700 Subject: [PATCH 5/5] create simplelog Logger interface --- d2cli/main.go | 23 ++++++---- lib/imgbundler/imgbundler.go | 28 ++++++------ lib/imgbundler/imgbundler_test.go | 25 ++++++----- lib/simplelog/simplelog.go | 71 +++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 33 deletions(-) create mode 100644 lib/simplelog/simplelog.go diff --git a/d2cli/main.go b/d2cli/main.go index 68ea60b42..52f3a2ed0 100644 --- a/d2cli/main.go +++ b/d2cli/main.go @@ -37,6 +37,7 @@ import ( "oss.terrastruct.com/d2/lib/pdf" "oss.terrastruct.com/d2/lib/png" "oss.terrastruct.com/d2/lib/pptx" + "oss.terrastruct.com/d2/lib/simplelog" "oss.terrastruct.com/d2/lib/textmeasure" timelib "oss.terrastruct.com/d2/lib/time" "oss.terrastruct.com/d2/lib/version" @@ -749,10 +750,11 @@ func _render(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opts } cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" - svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) + l := simplelog.FromCmdLog(ms.Log) + svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages) if bundle { var bundleErr2 error - svg, bundleErr2 = imgbundler.BundleRemote(ctx, svg, cacheImages) + svg, bundleErr2 = imgbundler.BundleRemote(ctx, l, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) } if forceAppendix && !toPNG { @@ -765,7 +767,7 @@ func _render(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opts if !bundle { var bundleErr2 error - svg, bundleErr2 = imgbundler.BundleRemote(ctx, svg, cacheImages) + svg, bundleErr2 = imgbundler.BundleRemote(ctx, l, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) } @@ -835,8 +837,9 @@ func renderPDF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opt } cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" - svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) - svg, bundleErr2 := imgbundler.BundleRemote(ctx, svg, cacheImages) + l := simplelog.FromCmdLog(ms.Log) + svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages) + svg, bundleErr2 := imgbundler.BundleRemote(ctx, l, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) if bundleErr != nil { return svg, bundleErr @@ -936,8 +939,9 @@ func renderPPTX(ctx context.Context, ms *xmain.State, presentation *pptx.Present } cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" - svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) - svg, bundleErr2 := imgbundler.BundleRemote(ctx, svg, cacheImages) + l := simplelog.FromCmdLog(ms.Log) + svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages) + svg, bundleErr2 := imgbundler.BundleRemote(ctx, l, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) if bundleErr != nil { return nil, bundleErr @@ -1182,8 +1186,9 @@ func renderPNGsForGIF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plug } cacheImages := ms.Env.Getenv("IMG_CACHE") == "1" - svg, bundleErr := imgbundler.BundleLocal(ctx, svg, cacheImages) - svg, bundleErr2 := imgbundler.BundleRemote(ctx, svg, cacheImages) + l := simplelog.FromCmdLog(ms.Log) + svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages) + svg, bundleErr2 := imgbundler.BundleRemote(ctx, l, svg, cacheImages) bundleErr = multierr.Combine(bundleErr, bundleErr2) if bundleErr != nil { return nil, nil, bundleErr diff --git a/lib/imgbundler/imgbundler.go b/lib/imgbundler/imgbundler.go index 8324ce205..08488df10 100644 --- a/lib/imgbundler/imgbundler.go +++ b/lib/imgbundler/imgbundler.go @@ -19,7 +19,7 @@ import ( "golang.org/x/xerrors" - "oss.terrastruct.com/d2/lib/log" + "oss.terrastruct.com/d2/lib/simplelog" "oss.terrastruct.com/util-go/xdefer" ) @@ -29,12 +29,12 @@ const maxImageSize int64 = 1 << 25 // 33_554_432 var imageRegex = regexp.MustCompile(` 0 { @@ -150,7 +150,7 @@ func runWorkers(ctx context.Context, svg []byte, imgs [][][]byte, isRemote, cach } } -func worker(ctx context.Context, href []byte, isRemote, cacheImages bool) ([]byte, error) { +func worker(ctx context.Context, l simplelog.Logger, href []byte, isRemote, cacheImages bool) ([]byte, error) { if cacheImages { if hit, ok := imgCache.Load(string(href)); ok { return hit.([]byte), nil @@ -160,10 +160,10 @@ func worker(ctx context.Context, href []byte, isRemote, cacheImages bool) ([]byt var mimeType string var err error if isRemote { - log.Debug(ctx, fmt.Sprintf("fetching %s remotely", string(href))) + l.Debug(fmt.Sprintf("fetching %s remotely", string(href))) buf, mimeType, err = httpGet(ctx, html.UnescapeString(string(href))) } else { - log.Debug(ctx, fmt.Sprintf("reading %s from disk", string(href))) + l.Debug(fmt.Sprintf("reading %s from disk", string(href))) buf, err = os.ReadFile(html.UnescapeString(string(href))) } if err != nil { diff --git a/lib/imgbundler/imgbundler_test.go b/lib/imgbundler/imgbundler_test.go index ae381636d..a8ebe37cd 100644 --- a/lib/imgbundler/imgbundler_test.go +++ b/lib/imgbundler/imgbundler_test.go @@ -16,6 +16,7 @@ import ( tassert "github.com/stretchr/testify/assert" "oss.terrastruct.com/d2/lib/log" + "oss.terrastruct.com/d2/lib/simplelog" ) //go:embed test_png.png @@ -96,7 +97,8 @@ width="328" height="587" viewBox="-100 -131 328 587"> `, svgURL, pngURL) - out, err := BundleLocal(ctx, []byte(sampleSVG), false) + l := simplelog.FromLibLog(ctx) + out, err := BundleLocal(ctx, l, []byte(sampleSVG), false) if err != nil { t.Fatal(err) } @@ -251,7 +254,8 @@ width="328" height="587" viewBox="-100 -131 328 587">