Merge pull request #278 from nhooyr/fixes-d6f3

imgbundler and CLI fixes
This commit is contained in:
Anmol Sethi 2022-11-30 18:00:29 -05:00 committed by GitHub
commit 29562feeb2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 221 additions and 156 deletions

View file

@ -12,6 +12,8 @@
- Querying shapes and connections by ID is now supported in renders. [#218](https://github.com/terrastruct/d2/pull/218) - Querying shapes and connections by ID is now supported in renders. [#218](https://github.com/terrastruct/d2/pull/218)
- [install.sh](./install.sh) now accepts `-d` as an alias for `--dry-run`. - [install.sh](./install.sh) now accepts `-d` as an alias for `--dry-run`.
[#266](https://github.com/terrastruct/d2/pull/266) [#266](https://github.com/terrastruct/d2/pull/266)
- `-b/--bundle` flag to `d2` now works and bundles all image assets directly as base64
data urls. [#278](https://github.com/terrastruct/d2/pull/278)
#### Improvements 🔧 #### Improvements 🔧

View file

@ -9,14 +9,7 @@
.Op Fl -watch Ar false .Op Fl -watch Ar false
.Op Fl -theme Em 0 .Op Fl -theme Em 0
.Ar file.d2 .Ar file.d2
.Op Ar file.svg .Op Ar file.svg | file.png
|
.Op Ar file.png
.Nm d2
.Op Fl -watch Ar false
.Op Fl -theme Em 0
.Ar file.d2
.Op Ar ...
.Nm d2 .Nm d2
.Ar layout Op Ar name .Ar layout Op Ar name
.Sh DESCRIPTION .Sh DESCRIPTION
@ -29,10 +22,21 @@ to
.Ar file.png .Ar file.png
.Ns . .Ns .
.Pp .Pp
It defaults to
.Ar file.svg
if no output path is passed.
.Pp
Pass - to have Pass - to have
.Nm .Nm
read from stdin or write to stdout. read from stdin or write to stdout.
.Pp .Pp
Never use the presence of the output file to check for success.
Always use the exit status of
.Nm d2
.Ns .
This is because sometimes when errors occur while rendering, d2 still write out a partial
render anyway to enable iteration on a broken diagram.
.Pp
See more docs, the source code and license at See more docs, the source code and license at
.Lk https://oss.terrastruct.com/d2 .Lk https://oss.terrastruct.com/d2
.Sh OPTIONS .Sh OPTIONS

View file

@ -17,9 +17,13 @@ func help(ms *xmain.State) {
fmt.Fprintf(ms.Stdout, `Usage: fmt.Fprintf(ms.Stdout, `Usage:
%s [--watch=false] [--theme=0] file.d2 [file.svg | file.png] %s [--watch=false] [--theme=0] file.d2 [file.svg | file.png]
%[1]s compiles and renders file.d2 to file.svg|file.png. %[1]s compiles and renders file.d2 to file.svg | file.png
It defaults to file.svg if an output path is not provided.
Use - to have d2 read from stdin or write to stdout. Use - to have d2 read from stdin or write to stdout.
See man %[1]s for more detailed docs.
Flags: Flags:
%s %s

View file

@ -12,6 +12,7 @@ import (
"github.com/playwright-community/playwright-go" "github.com/playwright-community/playwright-go"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"go.uber.org/multierr"
"oss.terrastruct.com/d2" "oss.terrastruct.com/d2"
"oss.terrastruct.com/d2/d2layouts/d2sequence" "oss.terrastruct.com/d2/d2layouts/d2sequence"
@ -37,26 +38,26 @@ func run(ctx context.Context, ms *xmain.State) (err error) {
// These should be kept up-to-date with the d2 man page // These should be kept up-to-date with the d2 man page
watchFlag, err := ms.Opts.Bool("D2_WATCH", "watch", "w", false, "watch for changes to input and live reload. Use $HOST and $PORT to specify the listening address.\n(default localhost:0, which is will open on a randomly available local port).") watchFlag, err := ms.Opts.Bool("D2_WATCH", "watch", "w", false, "watch for changes to input and live reload. Use $HOST and $PORT to specify the listening address.\n(default localhost:0, which is will open on a randomly available local port).")
if err != nil { if err != nil {
return xmain.UsageErrorf(err.Error()) return err
} }
hostFlag := ms.Opts.String("HOST", "host", "h", "localhost", "host listening address when used with watch") hostFlag := ms.Opts.String("HOST", "host", "h", "localhost", "host listening address when used with watch")
portFlag := ms.Opts.String("PORT", "port", "p", "0", "port listening address when used with watch") portFlag := ms.Opts.String("PORT", "port", "p", "0", "port listening address when used with watch")
bundleFlag, err := ms.Opts.Bool("D2_BUNDLE", "bundle", "b", true, "when outputting SVG, bundle all assets and layers into the output file.") bundleFlag, err := ms.Opts.Bool("D2_BUNDLE", "bundle", "b", true, "when outputting SVG, bundle all assets and layers into the output file.")
if err != nil { if err != nil {
return xmain.UsageErrorf(err.Error()) return err
} }
debugFlag, err := ms.Opts.Bool("DEBUG", "debug", "d", false, "print debug logs.") debugFlag, err := ms.Opts.Bool("DEBUG", "debug", "d", false, "print debug logs.")
if err != nil { if err != nil {
return xmain.UsageErrorf(err.Error()) return err
} }
layoutFlag := ms.Opts.String("D2_LAYOUT", "layout", "l", "dagre", `the layout engine used.`) layoutFlag := ms.Opts.String("D2_LAYOUT", "layout", "l", "dagre", `the layout engine used.`)
themeFlag, err := ms.Opts.Int64("D2_THEME", "theme", "t", 0, "the diagram theme ID. For a list of available options, see https://oss.terrastruct.com/d2") themeFlag, err := ms.Opts.Int64("D2_THEME", "theme", "t", 0, "the diagram theme ID. For a list of available options, see https://oss.terrastruct.com/d2")
if err != nil { if err != nil {
return xmain.UsageErrorf(err.Error()) return err
} }
versionFlag, err := ms.Opts.Bool("", "version", "v", false, "get the version") versionFlag, err := ms.Opts.Bool("", "version", "v", false, "get the version")
if err != nil { if err != nil {
return xmain.UsageErrorf(err.Error()) return err
} }
err = ms.Opts.Flags.Parse(ms.Opts.Args) err = ms.Opts.Flags.Parse(ms.Opts.Args)
@ -156,6 +157,7 @@ func run(ctx context.Context, ms *xmain.State) (err error) {
port: *portFlag, port: *portFlag,
inputPath: inputPath, inputPath: inputPath,
outputPath: outputPath, outputPath: outputPath,
bundle: *bundleFlag,
pw: pw, pw: pw,
}) })
if err != nil { if err != nil {
@ -167,27 +169,26 @@ func run(ctx context.Context, ms *xmain.State) (err error) {
ctx, cancel := context.WithTimeout(ctx, time.Minute*2) ctx, cancel := context.WithTimeout(ctx, time.Minute*2)
defer cancel() defer cancel()
if *bundleFlag { _, written, err := compile(ctx, ms, plugin, *themeFlag, inputPath, outputPath, *bundleFlag, pw.Page)
_ = 343
}
_, err = compile(ctx, ms, false, plugin, *themeFlag, inputPath, outputPath, pw.Page)
if err != nil { if err != nil {
return err if written {
return fmt.Errorf("failed to fully compile (partial render written): %w", err)
}
return fmt.Errorf("failed to compile: %w", err)
} }
ms.Log.Success.Printf("successfully compiled %v to %v", inputPath, outputPath) ms.Log.Success.Printf("successfully compiled %v to %v", inputPath, outputPath)
return nil return nil
} }
func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, page playwright.Page) ([]byte, error) { func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, bundle bool, page playwright.Page) (_ []byte, written bool, _ error) {
input, err := ms.ReadPath(inputPath) input, err := ms.ReadPath(inputPath)
if err != nil { if err != nil {
return nil, err return nil, false, err
} }
ruler, err := textmeasure.NewRuler() ruler, err := textmeasure.NewRuler()
if err != nil { if err != nil {
return nil, err return nil, false, err
} }
layout := plugin.Layout layout := plugin.Layout
@ -201,47 +202,46 @@ func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plu
ThemeID: themeID, ThemeID: themeID,
}) })
if err != nil { if err != nil {
return nil, err return nil, false, err
} }
svg, err := d2svg.Render(d) svg, err := d2svg.Render(d)
if err != nil { if err != nil {
return nil, err return nil, false, err
} }
svg, err = plugin.PostProcess(ctx, svg) svg, err = plugin.PostProcess(ctx, svg)
if err != nil { if err != nil {
return nil, err return svg, false, err
} }
svg, err = imgbundler.InlineLocal(ctx, ms, svg)
if err != nil { svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg)
ms.Log.Error.Printf("missing/broken local image(s), writing partial output: %v", err) if bundle {
var bundleErr2 error
svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg)
bundleErr = multierr.Combine(bundleErr, bundleErr2)
} }
out := svg out := svg
if filepath.Ext(outputPath) == ".png" { if filepath.Ext(outputPath) == ".png" {
svg, err = imgbundler.InlineRemote(ctx, ms, svg) svg := svg
if err != nil { if !bundle {
ms.Log.Error.Printf("missing/broken remote image(s), writing partial output: %v", err) var bundleErr2 error
svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg)
bundleErr = multierr.Combine(bundleErr, bundleErr2)
} }
out, err = png.ConvertSVG(ms, page, svg) out, err = png.ConvertSVG(ms, page, svg)
if err != nil { if err != nil {
return nil, err return svg, false, err
} }
} }
err = ms.WritePath(outputPath, out) err = ms.WritePath(outputPath, out)
if err != nil { if err != nil {
return nil, err return svg, false, err
} }
// Missing/broken images are fine during watch mode, as the user is likely building up a diagram. return svg, true, bundleErr
// Otherwise, the assumption is that this diagram is building for production, and broken images are not okay.
if !isWatching && ms.Log.Nerrors() > 0 {
return nil, xmain.ExitErrorf(1, "errors logged while rendering, partial output written to %v", outputPath)
}
return svg, nil
} }
// newExt must include leading . // newExt must include leading .

View file

@ -1,4 +1,4 @@
#d2c-err { #d2-err {
/* This style was copied from Chrome's svg parser error style. */ /* This style was copied from Chrome's svg parser error style. */
white-space: pre-wrap; white-space: pre-wrap;
border: 2px solid #c77; border: 2px solid #c77;

View file

@ -4,6 +4,9 @@ window.addEventListener("DOMContentLoaded", () => {
}); });
function init(reconnectDelay) { function init(reconnectDelay) {
const d2ErrDiv = window.document.querySelector("#d2-err");
const d2SVG = window.document.querySelector("#d2-svg");
const devMode = document.body.dataset.d2DevMode === "true"; const devMode = document.body.dataset.d2DevMode === "true";
const ws = new WebSocket( const ws = new WebSocket(
`ws://${window.location.host}${window.location.pathname}watch` `ws://${window.location.host}${window.location.pathname}watch`
@ -19,13 +22,7 @@ function init(reconnectDelay) {
} else { } else {
console.debug("watch websocket received data"); console.debug("watch websocket received data");
} }
const d2ErrDiv = window.document.querySelector("#d2-err"); if (msg.svg) {
if (msg.err) {
d2ErrDiv.innerText = msg.err;
d2ErrDiv.style.display = "block";
d2ErrDiv.scrollIntoView();
} else {
const d2SVG = window.document.querySelector("#d2-svg");
// We could turn d2SVG into an actual SVG element and use outerHTML to fully replace it // We could turn d2SVG into an actual SVG element and use outerHTML to fully replace it
// with the result from the renderer but unfortunately that overwrites the #d2-svg ID. // with the result from the renderer but unfortunately that overwrites the #d2-svg ID.
// Even if you add another line to set it afterwards. The parsing/interpretation of outerHTML must be async. // Even if you add another line to set it afterwards. The parsing/interpretation of outerHTML must be async.
@ -36,6 +33,11 @@ function init(reconnectDelay) {
d2SVG.innerHTML = msg.svg; d2SVG.innerHTML = msg.svg;
d2ErrDiv.style.display = "none"; d2ErrDiv.style.display = "none";
} }
if (msg.err) {
d2ErrDiv.innerText = msg.err;
d2ErrDiv.style.display = "block";
d2ErrDiv.scrollIntoView();
}
}; };
ws.onerror = (ev) => { ws.onerror = (ev) => {
console.error("watch websocket connection error", ev); console.error("watch websocket connection error", ev);

View file

@ -42,6 +42,7 @@ type watcherOpts struct {
port string port string
inputPath string inputPath string
outputPath string outputPath string
bundle bool
pw png.Playwright pw png.Playwright
} }
@ -73,8 +74,8 @@ type watcher struct {
} }
type compileResult struct { type compileResult struct {
Err string `json:"err"`
SVG string `json:"svg"` SVG string `json:"svg"`
Err string `json:"err"`
} }
func newWatcher(ctx context.Context, ms *xmain.State, opts watcherOpts) (*watcher, error) { func newWatcher(ctx context.Context, ms *xmain.State, opts watcherOpts) (*watcher, error) {
@ -345,19 +346,23 @@ func (w *watcher) compileLoop(ctx context.Context) error {
w.pw = newPW w.pw = newPW
} }
b, err := compile(ctx, w.ms, true, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.pw.Page) svg, _, err := compile(ctx, w.ms, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.bundle, w.pw.Page)
errs := ""
if err != nil { if err != nil {
if len(svg) > 0 {
err = fmt.Errorf("failed to fully %scompile (rendering partial svg): %w", recompiledPrefix, err)
} else {
err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err) err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err)
w.ms.Log.Error.Print(err) }
w.broadcast(&compileResult{ errs = err.Error()
Err: err.Error(), w.ms.Log.Error.Print(errs)
})
} else { } else {
w.ms.Log.Success.Printf("successfully %scompiled %v to %v", recompiledPrefix, w.inputPath, w.outputPath) w.ms.Log.Success.Printf("successfully %scompiled %v to %v", recompiledPrefix, w.inputPath, w.outputPath)
w.broadcast(&compileResult{
SVG: string(b),
})
} }
w.broadcast(&compileResult{
SVG: string(svg),
Err: errs,
})
if firstCompile { if firstCompile {
firstCompile = false firstCompile = false

2
go.mod generated
View file

@ -20,6 +20,7 @@ require (
golang.org/x/image v0.1.0 golang.org/x/image v0.1.0
golang.org/x/net v0.2.0 golang.org/x/net v0.2.0
golang.org/x/text v0.4.0 golang.org/x/text v0.4.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
gonum.org/v1/plot v0.12.0 gonum.org/v1/plot v0.12.0
nhooyr.io/websocket v1.8.7 nhooyr.io/websocket v1.8.7
oss.terrastruct.com/cmdlog v0.0.0-20221129200109-540ef52ff07d oss.terrastruct.com/cmdlog v0.0.0-20221129200109-540ef52ff07d
@ -57,7 +58,6 @@ require (
golang.org/x/crypto v0.3.0 // indirect golang.org/x/crypto v0.3.0 // indirect
golang.org/x/sys v0.2.0 // indirect golang.org/x/sys v0.2.0 // indirect
golang.org/x/term v0.2.0 // indirect golang.org/x/term v0.2.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
google.golang.org/protobuf v1.28.1 // indirect google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

View file

@ -6,15 +6,17 @@ import (
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"mime"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"path"
"regexp" "regexp"
"strings" "strings"
"sync" "sync"
"time" "time"
"go.uber.org/multierr" "golang.org/x/xerrors"
"oss.terrastruct.com/xdefer" "oss.terrastruct.com/xdefer"
"oss.terrastruct.com/d2/lib/xmain" "oss.terrastruct.com/d2/lib/xmain"
@ -22,140 +24,186 @@ import (
const maxImageSize int64 = 1 << 25 // 33_554_432 const maxImageSize int64 = 1 << 25 // 33_554_432
var imageRe = regexp.MustCompile(`<image href="([^"]+)"`) var imageRegex = regexp.MustCompile(`<image href="([^"]+)"`)
type resp struct { func BundleLocal(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
srctxt string return bundle(ctx, ms, in, false)
data string
err error
} }
func InlineLocal(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) { func BundleRemote(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
return inline(ctx, ms, in, false) return bundle(ctx, ms, in, true)
} }
func InlineRemote(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) { type repl struct {
return inline(ctx, ms, in, true) from []byte
to []byte
} }
func inline(ctx context.Context, ms *xmain.State, svg []byte, isRemote bool) (_ []byte, err error) { func bundle(ctx context.Context, ms *xmain.State, svg []byte, isRemote bool) (_ []byte, err error) {
defer xdefer.Errorf(&err, "failed to bundle images") if isRemote {
imgs := imageRe.FindAllSubmatch(svg, -1) defer xdefer.Errorf(&err, "failed to bundle remote images")
} else {
var filtered [][][]byte defer xdefer.Errorf(&err, "failed to bundle local images")
for _, img := range imgs {
u, err := url.Parse(string(img[1]))
isRemoteImg := err == nil && strings.HasPrefix(u.Scheme, "http")
if isRemoteImg == isRemote {
filtered = append(filtered, img)
} }
} imgs := imageRegex.FindAllSubmatch(svg, -1)
imgs = filterImageElements(imgs, isRemote)
var wg sync.WaitGroup
respChan := make(chan resp)
// Limits the number of workers to 16.
sema := make(chan struct{}, 16)
ctx, cancel := context.WithTimeout(ctx, time.Minute*5) ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
defer cancel() defer cancel()
wg.Add(len(filtered)) return runWorkers(ctx, ms, svg, imgs, isRemote)
}
// filterImageElements finds all image elements in imgs that are eligible
// for bundling in the current context.
func filterImageElements(imgs [][][]byte, isRemote bool) [][][]byte {
imgs2 := imgs[:0]
for _, img := range imgs {
href := string(img[1])
// Skip already bundled images.
if strings.HasPrefix(href, "data:") {
continue
}
u, err := url.Parse(href)
isRemoteImg := err == nil && strings.HasPrefix(u.Scheme, "http")
if isRemoteImg == isRemote {
imgs2 = append(imgs2, img)
}
}
return imgs2
}
func runWorkers(ctx context.Context, ms *xmain.State, svg []byte, imgs [][][]byte, isRemote bool) (_ []byte, err error) {
var wg sync.WaitGroup
replc := make(chan repl)
wg.Add(len(imgs))
go func() {
wg.Wait()
close(replc)
}()
// Limits the number of workers to 16.
sema := make(chan struct{}, 16)
var errhrefsMu sync.Mutex
var errhrefs []string
// Start workers as the sema allows. // Start workers as the sema allows.
go func() { go func() {
for _, img := range filtered { for _, img := range imgs {
img := img
sema <- struct{}{} sema <- struct{}{}
go func(src, href string) { go func() {
defer func() { defer func() {
wg.Done() wg.Done()
<-sema <-sema
}() }()
var data string bundledImage, err := worker(ctx, img[1], isRemote)
var err error if err != nil {
if isRemote { ms.Log.Error.Printf("failed to bundle %s: %v", img[1], err)
data, err = fetch(ctx, href) errhrefsMu.Lock()
} else { errhrefs = append(errhrefs, string(img[1]))
data, err = read(href) errhrefsMu.Unlock()
return
} }
select { select {
case <-ctx.Done(): case <-ctx.Done():
case respChan <- resp{ case replc <- repl{
srctxt: src, from: img[0],
data: data, to: bundledImage,
err: err,
}: }:
} }
}(string(img[0]), string(img[1])) }()
} }
}() }()
go func() { t := time.NewTicker(time.Second * 5)
wg.Wait() defer t.Stop()
close(respChan)
}()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return nil, fmt.Errorf("failed to wait for imgbundler workers: %w", ctx.Err()) return svg, xerrors.Errorf("failed to wait for workers: %w", ctx.Err())
case <-time.After(time.Second * 5): case <-t.C:
ms.Log.Info.Printf("fetching images...") ms.Log.Info.Printf("fetching images...")
case resp, ok := <-respChan: case repl, ok := <-replc:
if !ok { if !ok {
return svg, err if len(errhrefs) > 0 {
return svg, xerrors.Errorf("%v", errhrefs)
} }
if resp.err != nil { return svg, nil
err = multierr.Combine(err, resp.err)
continue
} }
svg = bytes.Replace(svg, []byte(resp.srctxt), []byte(fmt.Sprintf(`<image href="%s"`, resp.data)), 1) svg = bytes.Replace(svg, repl.from, repl.to, 1)
} }
} }
} }
var imgClient = &http.Client{} func worker(ctx context.Context, href []byte, isRemote bool) ([]byte, error) {
var buf []byte
var mimeType string
var err error
if isRemote {
buf, mimeType, err = httpGet(ctx, string(href))
} else {
buf, err = os.ReadFile(string(href))
}
if err != nil {
return nil, err
}
func fetch(ctx context.Context, href string) (string, error) { if mimeType == "" {
mimeType = sniffMimeType(href, buf, isRemote)
}
mimeType = strings.Replace(mimeType, "text/xml", "image/svg+xml", 1)
b64 := base64.StdEncoding.EncodeToString(buf)
return []byte(fmt.Sprintf(`<image href="data:%s;base64,%s"`, mimeType, b64)), nil
}
var httpClient = &http.Client{}
func httpGet(ctx context.Context, href string) ([]byte, string, error) {
ctx, cancel := context.WithTimeout(ctx, time.Minute) ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel() defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", href, nil) req, err := http.NewRequestWithContext(ctx, "GET", href, nil)
if err != nil { if err != nil {
return "", err return nil, "", err
} }
imgResp, err := imgClient.Do(req) resp, err := httpClient.Do(req)
if err != nil { if err != nil {
return "", err return nil, "", err
} }
defer imgResp.Body.Close() defer resp.Body.Close()
if imgResp.StatusCode != 200 { if resp.StatusCode != 200 {
return "", fmt.Errorf("img %s returned status code %d", href, imgResp.StatusCode) return nil, "", fmt.Errorf("expected status 200 but got %d %s", resp.StatusCode, resp.Status)
} }
r := http.MaxBytesReader(nil, imgResp.Body, maxImageSize) r := http.MaxBytesReader(nil, resp.Body, maxImageSize)
data, err := ioutil.ReadAll(r) buf, err := ioutil.ReadAll(r)
if err != nil { if err != nil {
return "", err return nil, "", err
}
return buf, resp.Header.Get("Content-Type"), nil
} }
mimeType := http.DetectContentType(data) // sniffMimeType sniffs the mime type of href based on its file extension and contents.
mimeType = strings.Replace(mimeType, "text/xml", "image/svg+xml", 1) func sniffMimeType(href, buf []byte, isRemote bool) string {
p := string(href)
enc := base64.StdEncoding.EncodeToString(data) if isRemote {
u, err := url.Parse(p)
return fmt.Sprintf("data:%s;base64,%s", mimeType, enc), nil
}
func read(href string) (string, error) {
data, err := os.ReadFile(href)
if err != nil { if err != nil {
return "", err p = ""
} else {
p = u.Path
} }
}
mimeType := http.DetectContentType(data) mimeType := mime.TypeByExtension(path.Ext(p))
mimeType = strings.Replace(mimeType, "text/xml", "image/svg+xml", 1) if mimeType == "" {
mimeType = http.DetectContentType(buf)
enc := base64.StdEncoding.EncodeToString(data) }
return mimeType
return fmt.Sprintf("data:%s;base64,%s", mimeType, enc), nil
} }

View file

@ -41,7 +41,7 @@ func TestRegex(t *testing.T) {
for _, href := range append(urls, notURLs...) { for _, href := range append(urls, notURLs...) {
str := fmt.Sprintf(`<image href="%s" />`, href) str := fmt.Sprintf(`<image href="%s" />`, href)
matches := imageRe.FindAllStringSubmatch(str, -1) matches := imageRegex.FindAllStringSubmatch(str, -1)
if len(matches) != 1 { if len(matches) != 1 {
t.Fatalf("uri regex didn't match %s", str) t.Fatalf("uri regex didn't match %s", str)
} }
@ -90,7 +90,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
} }
ms.Log = cmdlog.Log(ms.Env, os.Stderr) ms.Log = cmdlog.Log(ms.Env, os.Stderr)
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response { httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
respRecorder := httptest.NewRecorder() respRecorder := httptest.NewRecorder()
switch req.URL.String() { switch req.URL.String() {
case svgURL: case svgURL:
@ -104,7 +104,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
return respRecorder.Result() return respRecorder.Result()
}) })
out, err := InlineRemote(ctx, ms, []byte(sampleSVG)) out, err := BundleRemote(ctx, ms, []byte(sampleSVG))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -119,7 +119,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
} }
// Test almost too large response // Test almost too large response
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response { httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
respRecorder := httptest.NewRecorder() respRecorder := httptest.NewRecorder()
bytes := make([]byte, maxImageSize) bytes := make([]byte, maxImageSize)
rand.Read(bytes) rand.Read(bytes)
@ -127,13 +127,13 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
respRecorder.WriteHeader(200) respRecorder.WriteHeader(200)
return respRecorder.Result() return respRecorder.Result()
}) })
_, err = InlineRemote(ctx, ms, []byte(sampleSVG)) _, err = BundleRemote(ctx, ms, []byte(sampleSVG))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Test too large response // Test too large response
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response { httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
respRecorder := httptest.NewRecorder() respRecorder := httptest.NewRecorder()
bytes := make([]byte, maxImageSize+1) bytes := make([]byte, maxImageSize+1)
rand.Read(bytes) rand.Read(bytes)
@ -141,18 +141,18 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
respRecorder.WriteHeader(200) respRecorder.WriteHeader(200)
return respRecorder.Result() return respRecorder.Result()
}) })
_, err = InlineRemote(ctx, ms, []byte(sampleSVG)) _, err = BundleRemote(ctx, ms, []byte(sampleSVG))
if err == nil { if err == nil {
t.Fatal("expected error") t.Fatal("expected error")
} }
// Test error response // Test error response
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response { httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
respRecorder := httptest.NewRecorder() respRecorder := httptest.NewRecorder()
respRecorder.WriteHeader(500) respRecorder.WriteHeader(500)
return respRecorder.Result() return respRecorder.Result()
}) })
_, err = InlineRemote(ctx, ms, []byte(sampleSVG)) _, err = BundleRemote(ctx, ms, []byte(sampleSVG))
if err == nil { if err == nil {
t.Fatal("expected error") t.Fatal("expected error")
} }
@ -205,7 +205,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
Env: xos.NewEnv(os.Environ()), Env: xos.NewEnv(os.Environ()),
} }
ms.Log = cmdlog.Log(ms.Env, os.Stderr) ms.Log = cmdlog.Log(ms.Env, os.Stderr)
out, err := InlineLocal(ctx, ms, []byte(sampleSVG)) out, err := BundleLocal(ctx, ms, []byte(sampleSVG))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View file

@ -129,7 +129,7 @@ func (o *Opts) Int64(envKey, flag, shortFlag string, defaultVal int64, usage str
if env := o.getEnv(flag, envKey); env != "" { if env := o.getEnv(flag, envKey); env != "" {
envVal, err := strconv.ParseInt(env, 10, 64) envVal, err := strconv.ParseInt(env, 10, 64)
if err != nil { if err != nil {
return nil, fmt.Errorf(`invalid environment variable %s. Expected int64. Found "%v".`, envKey, envVal) return nil, UsageErrorf(`invalid environment variable %s. Expected int64. Found "%v".`, envKey, envVal)
} }
defaultVal = envVal defaultVal = envVal
} }
@ -148,7 +148,7 @@ func (o *Opts) String(envKey, flag, shortFlag string, defaultVal, usage string)
func (o *Opts) Bool(envKey, flag, shortFlag string, defaultVal bool, usage string) (*bool, error) { func (o *Opts) Bool(envKey, flag, shortFlag string, defaultVal bool, usage string) (*bool, error) {
if env := o.getEnv(flag, envKey); env != "" { if env := o.getEnv(flag, envKey); env != "" {
if !boolyEnv(env) { if !boolyEnv(env) {
return nil, fmt.Errorf(`invalid environment variable %s. Expected bool. Found "%s".`, envKey, env) return nil, UsageErrorf(`invalid environment variable %s. Expected bool. Found "%s".`, envKey, env)
} }
if truthyEnv(env) { if truthyEnv(env) {
defaultVal = true defaultVal = true