luciano/d2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1678 from gavin-ts/update-imgbundler

Browse source 
remove imgbundler's xmain.State dependency
This commit is contained in:
gavin-ts 2023-10-19 21:24:36 -07:00 committed by GitHub
commit 4cba5d27e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 153 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
d2cli/main.go
View file

@ -37,6 +37,7 @@ import (
"oss.terrastruct.com/d2/lib/pdf"
"oss.terrastruct.com/d2/lib/png"
"oss.terrastruct.com/d2/lib/pptx"
"oss.terrastruct.com/d2/lib/simplelog"
"oss.terrastruct.com/d2/lib/textmeasure"
timelib "oss.terrastruct.com/d2/lib/time"
"oss.terrastruct.com/d2/lib/version"
@ -434,7 +435,7 @@ func compile(ctx context.Context, ms *xmain.State, plugins []d2plugin.Plugin, la
if err != nil {
return nil, false, err
}
out, err := xgif.AnimatePNGs(ms, pngs, int(animateInterval))
out, err := AnimatePNGs(ms, pngs, int(animateInterval))
if err != nil {
return nil, false, err
}
@ -748,10 +749,12 @@ func _render(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opts
return svg, err
}
svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg)
cacheImages := ms.Env.Getenv("IMG_CACHE") == "1"
l := simplelog.FromCmdLog(ms.Log)
svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages)
if bundle {
var bundleErr2 error
svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg)
svg, bundleErr2 = imgbundler.BundleRemote(ctx, l, svg, cacheImages)
bundleErr = multierr.Combine(bundleErr, bundleErr2)
}
if forceAppendix && !toPNG {
@ -764,11 +767,11 @@ func _render(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opts
if !bundle {
var bundleErr2 error
svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg)
svg, bundleErr2 = imgbundler.BundleRemote(ctx, l, svg, cacheImages)
bundleErr = multierr.Combine(bundleErr, bundleErr2)
}
out, err = png.ConvertSVG(ms, page, svg)
out, err = ConvertSVG(ms, page, svg)
if err != nil {
return svg, err
}
@ -833,15 +836,17 @@ func renderPDF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, opt
return svg, err
}
svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg)
svg, bundleErr2 := imgbundler.BundleRemote(ctx, ms, svg)
cacheImages := ms.Env.Getenv("IMG_CACHE") == "1"
l := simplelog.FromCmdLog(ms.Log)
svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages)
svg, bundleErr2 := imgbundler.BundleRemote(ctx, l, svg, cacheImages)
bundleErr = multierr.Combine(bundleErr, bundleErr2)
if bundleErr != nil {
return svg, bundleErr
}
svg = appendix.Append(diagram, ruler, svg)
pngImg, err := png.ConvertSVG(ms, page, svg)
pngImg, err := ConvertSVG(ms, page, svg)
if err != nil {
return svg, err
}
@ -933,8 +938,10 @@ func renderPPTX(ctx context.Context, ms *xmain.State, presentation *pptx.Present
return nil, err
}
svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg)
svg, bundleErr2 := imgbundler.BundleRemote(ctx, ms, svg)
cacheImages := ms.Env.Getenv("IMG_CACHE") == "1"
l := simplelog.FromCmdLog(ms.Log)
svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages)
svg, bundleErr2 := imgbundler.BundleRemote(ctx, l, svg, cacheImages)
bundleErr = multierr.Combine(bundleErr, bundleErr2)
if bundleErr != nil {
return nil, bundleErr
@ -942,7 +949,7 @@ func renderPPTX(ctx context.Context, ms *xmain.State, presentation *pptx.Present
svg = appendix.Append(diagram, ruler, svg)
pngImg, err := png.ConvertSVG(ms, page, svg)
pngImg, err := ConvertSVG(ms, page, svg)
if err != nil {
return nil, err
}
@ -1178,8 +1185,10 @@ func renderPNGsForGIF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plug
return nil, nil, err
}
svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg)
svg, bundleErr2 := imgbundler.BundleRemote(ctx, ms, svg)
cacheImages := ms.Env.Getenv("IMG_CACHE") == "1"
l := simplelog.FromCmdLog(ms.Log)
svg, bundleErr := imgbundler.BundleLocal(ctx, l, svg, cacheImages)
svg, bundleErr2 := imgbundler.BundleRemote(ctx, l, svg, cacheImages)
bundleErr = multierr.Combine(bundleErr, bundleErr2)
if bundleErr != nil {
return nil, nil, bundleErr
@ -1187,7 +1196,7 @@ func renderPNGsForGIF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plug
svg = appendix.Append(diagram, ruler, svg)
pngImg, err := png.ConvertSVG(ms, page, svg)
pngImg, err := ConvertSVG(ms, page, svg)
if err != nil {
return nil, nil, err
}
@ -1218,3 +1227,21 @@ func renderPNGsForGIF(ctx context.Context, ms *xmain.State, plugin d2plugin.Plug
return svg, pngs, nil
}
func ConvertSVG(ms *xmain.State, page playwright.Page, svg []byte) ([]byte, error) {
cancel := background.Repeat(func() {
ms.Log.Info.Printf("converting to PNG...")
}, time.Second*5)
defer cancel()
return png.ConvertSVG(page, svg)
}
func AnimatePNGs(ms *xmain.State, pngs [][]byte, animIntervalMs int) ([]byte, error) {
cancel := background.Repeat(func() {
ms.Log.Info.Printf("generating GIF...")
}, time.Second*5)
defer cancel()
return xgif.AnimatePNGs(pngs, animIntervalMs)
}

32
lib/imgbundler/imgbundler.go
View file

@ -19,8 +19,8 @@ import (
"golang.org/x/xerrors"
"oss.terrastruct.com/d2/lib/simplelog"
"oss.terrastruct.com/util-go/xdefer"
"oss.terrastruct.com/util-go/xmain"
)
var imgCache sync.Map
@ -29,12 +29,12 @@ const maxImageSize int64 = 1 << 25 // 33_554_432
var imageRegex = regexp.MustCompile(`<image href="([^"]+)"`)
func BundleLocal(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
return bundle(ctx, ms, in, false)
func BundleLocal(ctx context.Context, l simplelog.Logger, in []byte, cacheImages bool) ([]byte, error) {
return bundle(ctx, l, in, false, cacheImages)
}
func BundleRemote(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
return bundle(ctx, ms, in, true)
func BundleRemote(ctx context.Context, l simplelog.Logger, in []byte, cacheImages bool) ([]byte, error) {
return bundle(ctx, l, in, true, cacheImages)
}
type repl struct {
@ -42,7 +42,7 @@ type repl struct {
to []byte
}
func bundle(ctx context.Context, ms *xmain.State, svg []byte, isRemote bool) (_ []byte, err error) {
func bundle(ctx context.Context, l simplelog.Logger, svg []byte, isRemote, cacheImages bool) (_ []byte, err error) {
if isRemote {
defer xdefer.Errorf(&err, "failed to bundle remote images")
} else {
@ -54,7 +54,7 @@ func bundle(ctx context.Context, ms *xmain.State, svg []byte, isRemote bool) (_
ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
defer cancel()
return runWorkers(ctx, ms, svg, imgs, isRemote)
return runWorkers(ctx, l, svg, imgs, isRemote, cacheImages)
}
// filterImageElements finds all unique image elements in imgs that are
@ -84,7 +84,7 @@ func filterImageElements(imgs [][][]byte, isRemote bool) [][][]byte {
return imgs2
}
func runWorkers(ctx context.Context, ms *xmain.State, svg []byte, imgs [][][]byte, isRemote bool) (_ []byte, err error) {
func runWorkers(ctx context.Context, l simplelog.Logger, svg []byte, imgs [][][]byte, isRemote, cacheImages bool) (_ []byte, err error) {
var wg sync.WaitGroup
replc := make(chan repl)
@ -111,9 +111,9 @@ func runWorkers(ctx context.Context, ms *xmain.State, svg []byte, imgs [][][]byt
<-sema
}()
bundledImage, err := worker(ctx, ms, img[1], isRemote)
bundledImage, err := worker(ctx, l, img[1], isRemote, cacheImages)
if err != nil {
ms.Log.Error.Printf("failed to bundle %s: %v", img[1], err)
l.Error(fmt.Sprintf("failed to bundle %s: %v", img[1], err))
errhrefsMu.Lock()
errhrefs = append(errhrefs, string(img[1]))
errhrefsMu.Unlock()
@ -137,7 +137,7 @@ func runWorkers(ctx context.Context, ms *xmain.State, svg []byte, imgs [][][]byt
case <-ctx.Done():
return svg, xerrors.Errorf("failed to wait for workers: %w", ctx.Err())
case <-t.C:
ms.Log.Info.Printf("fetching images...")
l.Info("fetching images...")
case repl, ok := <-replc:
if !ok {
if len(errhrefs) > 0 {
@ -150,8 +150,8 @@ func runWorkers(ctx context.Context, ms *xmain.State, svg []byte, imgs [][][]byt
}
}
func worker(ctx context.Context, ms *xmain.State, href []byte, isRemote bool) ([]byte, error) {
if ms.Env.Getenv("IMG_CACHE") == "1" {
func worker(ctx context.Context, l simplelog.Logger, href []byte, isRemote, cacheImages bool) ([]byte, error) {
if cacheImages {
if hit, ok := imgCache.Load(string(href)); ok {
return hit.([]byte), nil
}
@ -160,10 +160,10 @@ func worker(ctx context.Context, ms *xmain.State, href []byte, isRemote bool) ([
var mimeType string
var err error
if isRemote {
ms.Log.Debug.Printf("fetching %s remotely", string(href))
l.Debug(fmt.Sprintf("fetching %s remotely", string(href)))
buf, mimeType, err = httpGet(ctx, html.UnescapeString(string(href)))
} else {
ms.Log.Debug.Printf("reading %s from disk", string(href))
l.Debug(fmt.Sprintf("reading %s from disk", string(href)))
buf, err = os.ReadFile(html.UnescapeString(string(href)))
}
if err != nil {
@ -177,7 +177,7 @@ func worker(ctx context.Context, ms *xmain.State, href []byte, isRemote bool) ([
b64 := base64.StdEncoding.EncodeToString(buf)
out := []byte(fmt.Sprintf(`<image href="data:%s;base64,%s"`, mimeType, b64))
if ms.Env.Getenv("IMG_CACHE") == "1" {
if cacheImages {
imgCache.Store(string(href), out)
}
return out, nil

86
lib/imgbundler/imgbundler_test.go
View file

@ -7,18 +7,16 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"cdr.dev/slog/sloggers/slogtest"
tassert "github.com/stretchr/testify/assert"
"oss.terrastruct.com/util-go/cmdlog"
"oss.terrastruct.com/util-go/xos"
"oss.terrastruct.com/util-go/xmain"
"oss.terrastruct.com/d2/lib/log"
"oss.terrastruct.com/d2/lib/simplelog"
)
//go:embed test_png.png
@ -53,7 +51,8 @@ func TestRegex(t *testing.T) {
func TestInlineRemote(t *testing.T) {
imgCache = sync.Map{}
ctx := context.Background()
// we don't want log.Error to cause this test to fail
ctx := log.WithTB(context.Background(), t, &slogtest.Options{IgnoreErrors: true})
svgURL := "https://icons.terrastruct.com/essentials/004-picture.svg"
pngURL := "https://cdn4.iconfinder.com/data/icons/smart-phones-technologies/512/android-phone.png"
@ -84,17 +83,6 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
}]]></style></svg>
`, svgURL, pngURL)
ms := &xmain.State{
Name: "test",
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
Env: xos.NewEnv(os.Environ()),
}
ms.Log = cmdlog.NewTB(ms.Env, t)
httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
respRecorder := httptest.NewRecorder()
switch req.URL.String() {
@ -109,7 +97,8 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
return respRecorder.Result()
})
out, err := BundleRemote(ctx, ms, []byte(sampleSVG))
l := simplelog.FromLibLog(ctx)
out, err := BundleRemote(ctx, l, []byte(sampleSVG), false)
if err != nil {
t.Fatal(err)
}
@ -133,7 +122,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
respRecorder.WriteHeader(200)
return respRecorder.Result()
})
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
_, err = BundleRemote(ctx, l, []byte(sampleSVG), false)
if err != nil {
t.Fatal(err)
}
@ -148,7 +137,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
respRecorder.WriteHeader(200)
return respRecorder.Result()
})
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
_, err = BundleRemote(ctx, l, []byte(sampleSVG), false)
if err == nil {
t.Fatal("expected error")
}
@ -160,7 +149,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
respRecorder.WriteHeader(500)
return respRecorder.Result()
})
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
_, err = BundleRemote(ctx, l, []byte(sampleSVG), false)
if err == nil {
t.Fatal("expected error")
}
@ -168,7 +157,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
func TestInlineLocal(t *testing.T) {
imgCache = sync.Map{}
ctx := context.Background()
ctx := log.WithTB(context.Background(), t, nil)
svgURL, err := filepath.Abs("./test_svg.svg")
if err != nil {
t.Fatal(err)
@ -205,17 +194,8 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
}]]></style></svg>
`, svgURL, pngURL)
ms := &xmain.State{
Name: "test",
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
Env: xos.NewEnv(os.Environ()),
}
ms.Log = cmdlog.NewTB(ms.Env, t)
out, err := BundleLocal(ctx, ms, []byte(sampleSVG))
l := simplelog.FromLibLog(ctx)
out, err := BundleLocal(ctx, l, []byte(sampleSVG), false)
if err != nil {
t.Fatal(err)
}
@ -233,7 +213,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
// TestDuplicateURL ensures that we don't fetch the same image twice
func TestDuplicateURL(t *testing.T) {
imgCache = sync.Map{}
ctx := context.Background()
ctx := log.WithTB(context.Background(), t, nil)
url1 := "https://icons.terrastruct.com/essentials/004-picture.svg"
url2 := "https://icons.terrastruct.com/essentials/004-picture.svg"
@ -264,17 +244,6 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
}]]></style></svg>
`, url1, url2)
ms := &xmain.State{
Name: "test",
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
Env: xos.NewEnv(os.Environ()),
}
ms.Log = cmdlog.NewTB(ms.Env, t)
count := 0
httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
@ -285,7 +254,8 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
return respRecorder.Result()
})
out, err := BundleRemote(ctx, ms, []byte(sampleSVG))
l := simplelog.FromLibLog(ctx)
out, err := BundleRemote(ctx, l, []byte(sampleSVG), false)
if err != nil {
t.Fatal(err)
}
@ -298,7 +268,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
func TestImgCache(t *testing.T) {
imgCache = sync.Map{}
ctx := context.Background()
ctx := log.WithTB(context.Background(), t, nil)
url1 := "https://icons.terrastruct.com/essentials/004-picture.svg"
url2 := "https://icons.terrastruct.com/essentials/004-picture.svg"
@ -329,17 +299,6 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
}]]></style></svg>
`, url1, url2)
ms := &xmain.State{
Name: "test",
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
Env: xos.NewEnv(os.Environ()),
}
ms.Log = cmdlog.NewTB(ms.Env, t)
count := 0
httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
@ -350,26 +309,25 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
return respRecorder.Result()
})
l := simplelog.FromLibLog(ctx)
// Using a cache, imgs are not refetched on multiple runs
ms.Env.Setenv("IMG_CACHE", "1")
_, err := BundleRemote(ctx, ms, []byte(sampleSVG))
_, err := BundleRemote(ctx, l, []byte(sampleSVG), true)
if err != nil {
t.Fatal(err)
}
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
_, err = BundleRemote(ctx, l, []byte(sampleSVG), true)
if err != nil {
t.Fatal(err)
}
tassert.Equal(t, 1, count)
// With cache disabled, it refetches
ms.Env.Setenv("IMG_CACHE", "0")
count = 0
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
_, err = BundleRemote(ctx, l, []byte(sampleSVG), false)
if err != nil {
t.Fatal(err)
}
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
_, err = BundleRemote(ctx, l, []byte(sampleSVG), false)
if err != nil {
t.Fatal(err)
}

10
lib/png/png.go
View file

@ -5,7 +5,6 @@ import (
"encoding/base64"
"fmt"
"strings"
"time"
_ "embed"
@ -14,9 +13,7 @@ import (
pngstruct "github.com/dsoprea/go-png-image-structure/v2"
"github.com/playwright-community/playwright-go"
"oss.terrastruct.com/d2/lib/background"
"oss.terrastruct.com/d2/lib/version"
"oss.terrastruct.com/util-go/xmain"
)
// ConvertSVG scales the image by 2x
@ -88,12 +85,7 @@ const pngPrefix = "data:image/png;base64,"
// ConvertSVG converts the given SVG into a PNG.
// Note that the resulting PNG has 2x the size (width and height) of the original SVG (see generate_png.js)
func ConvertSVG(ms *xmain.State, page playwright.Page, svg []byte) ([]byte, error) {
cancel := background.Repeat(func() {
ms.Log.Info.Printf("converting to PNG...")
}, time.Second*5)
defer cancel()
func ConvertSVG(page playwright.Page, svg []byte) ([]byte, error) {
encodedSVG := base64.StdEncoding.EncodeToString(svg)
pngInterface, err := page.Evaluate(genPNGScript, map[string]interface{}{
"imgString": "data:image/svg+xml;charset=utf-8;base64," + encodedSVG,

71
lib/simplelog/simplelog.go Normal file
View file

@ -0,0 +1,71 @@
// Package simplelog contains a very simple interface for logging strings at either Debug, Info, or Error levels
package simplelog
import (
"context"
"oss.terrastruct.com/d2/lib/log"
"oss.terrastruct.com/util-go/cmdlog"
)
type Logger interface {
Debug(string)
Info(string)
Error(string)
}
type logger struct {
logDebug *func(string)
logInfo *func(string)
logError *func(string)
}
func (l logger) Debug(s string) {
if l.logDebug != nil {
(*l.logDebug)(s)
}
}
func (l logger) Info(s string) {
if l.logInfo != nil {
(*l.logInfo)(s)
}
}
func (l logger) Error(s string) {
if l.logError != nil {
(*l.logError)(s)
}
}
func Make(logDebug, logInfo, logError *func(string)) Logger {
return logger{
logDebug: logDebug,
logInfo: logInfo,
logError: logError,
}
}
func FromLibLog(ctx context.Context) Logger {
lDebug := func(s string) {
log.Debug(ctx, s)
}
lInfo := func(s string) {
log.Info(ctx, s)
}
lError := func(s string) {
log.Error(ctx, s)
}
return Make(&lDebug, &lInfo, &lError)
}
func FromCmdLog(cl *cmdlog.Logger) Logger {
lDebug := func(s string) {
cl.Debug.Print(s)
}
lInfo := func(s string) {
cl.Info.Print(s)
}
lError := func(s string) {
cl.Error.Print(s)
}
return Make(&lDebug, &lInfo, &lError)
}

12
lib/xgif/xgif.go
View file

@ -15,27 +15,17 @@ import (
"image/color"
"image/gif"
"image/png"
"time"
"github.com/ericpauley/go-quantize/quantize"
"oss.terrastruct.com/d2/lib/background"
"oss.terrastruct.com/util-go/go2"
"oss.terrastruct.com/util-go/xmain"
)
const INFINITE_LOOP = 0
var BG_COLOR = color.White
func AnimatePNGs(ms *xmain.State, pngs [][]byte, animIntervalMs int) ([]byte, error) {
if ms != nil {
cancel := background.Repeat(func() {
ms.Log.Info.Printf("generating GIF...")
}, time.Second*5)
defer cancel()
}
func AnimatePNGs(pngs [][]byte, animIntervalMs int) ([]byte, error) {
var width, height int
pngImgs := make([]image.Image, len(pngs))
for i, pngBytes := range pngs {

2
lib/xgif/xgif_test.go
View file

@ -20,7 +20,7 @@ var test_output []byte
func TestPngToGif(t *testing.T) {
boards := [][]byte{test_input1, test_input2}
interval := 1_000
gifBytes, err := AnimatePNGs(nil, boards, interval)
gifBytes, err := AnimatePNGs(boards, interval)
assert.NoError(t, err)
// use this to update the test output