Merge pull request #278 from nhooyr/fixes-d6f3
imgbundler and CLI fixes
This commit is contained in:
commit
29562feeb2
11 changed files with 221 additions and 156 deletions
|
|
@ -12,6 +12,8 @@
|
||||||
- Querying shapes and connections by ID is now supported in renders. [#218](https://github.com/terrastruct/d2/pull/218)
|
- Querying shapes and connections by ID is now supported in renders. [#218](https://github.com/terrastruct/d2/pull/218)
|
||||||
- [install.sh](./install.sh) now accepts `-d` as an alias for `--dry-run`.
|
- [install.sh](./install.sh) now accepts `-d` as an alias for `--dry-run`.
|
||||||
[#266](https://github.com/terrastruct/d2/pull/266)
|
[#266](https://github.com/terrastruct/d2/pull/266)
|
||||||
|
- `-b/--bundle` flag to `d2` now works and bundles all image assets directly as base64
|
||||||
|
data urls. [#278](https://github.com/terrastruct/d2/pull/278)
|
||||||
|
|
||||||
#### Improvements 🔧
|
#### Improvements 🔧
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,14 +9,7 @@
|
||||||
.Op Fl -watch Ar false
|
.Op Fl -watch Ar false
|
||||||
.Op Fl -theme Em 0
|
.Op Fl -theme Em 0
|
||||||
.Ar file.d2
|
.Ar file.d2
|
||||||
.Op Ar file.svg
|
.Op Ar file.svg | file.png
|
||||||
|
|
|
||||||
.Op Ar file.png
|
|
||||||
.Nm d2
|
|
||||||
.Op Fl -watch Ar false
|
|
||||||
.Op Fl -theme Em 0
|
|
||||||
.Ar file.d2
|
|
||||||
.Op Ar ...
|
|
||||||
.Nm d2
|
.Nm d2
|
||||||
.Ar layout Op Ar name
|
.Ar layout Op Ar name
|
||||||
.Sh DESCRIPTION
|
.Sh DESCRIPTION
|
||||||
|
|
@ -29,10 +22,21 @@ to
|
||||||
.Ar file.png
|
.Ar file.png
|
||||||
.Ns .
|
.Ns .
|
||||||
.Pp
|
.Pp
|
||||||
|
It defaults to
|
||||||
|
.Ar file.svg
|
||||||
|
if no output path is passed.
|
||||||
|
.Pp
|
||||||
Pass - to have
|
Pass - to have
|
||||||
.Nm
|
.Nm
|
||||||
read from stdin or write to stdout.
|
read from stdin or write to stdout.
|
||||||
.Pp
|
.Pp
|
||||||
|
Never use the presence of the output file to check for success.
|
||||||
|
Always use the exit status of
|
||||||
|
.Nm d2
|
||||||
|
.Ns .
|
||||||
|
This is because sometimes when errors occur while rendering, d2 still write out a partial
|
||||||
|
render anyway to enable iteration on a broken diagram.
|
||||||
|
.Pp
|
||||||
See more docs, the source code and license at
|
See more docs, the source code and license at
|
||||||
.Lk https://oss.terrastruct.com/d2
|
.Lk https://oss.terrastruct.com/d2
|
||||||
.Sh OPTIONS
|
.Sh OPTIONS
|
||||||
|
|
|
||||||
|
|
@ -15,11 +15,15 @@ import (
|
||||||
|
|
||||||
func help(ms *xmain.State) {
|
func help(ms *xmain.State) {
|
||||||
fmt.Fprintf(ms.Stdout, `Usage:
|
fmt.Fprintf(ms.Stdout, `Usage:
|
||||||
%s [--watch=false] [--theme=0] file.d2 [file.svg|file.png]
|
%s [--watch=false] [--theme=0] file.d2 [file.svg | file.png]
|
||||||
|
|
||||||
|
%[1]s compiles and renders file.d2 to file.svg | file.png
|
||||||
|
It defaults to file.svg if an output path is not provided.
|
||||||
|
|
||||||
%[1]s compiles and renders file.d2 to file.svg|file.png.
|
|
||||||
Use - to have d2 read from stdin or write to stdout.
|
Use - to have d2 read from stdin or write to stdout.
|
||||||
|
|
||||||
|
See man %[1]s for more detailed docs.
|
||||||
|
|
||||||
Flags:
|
Flags:
|
||||||
%s
|
%s
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import (
|
||||||
|
|
||||||
"github.com/playwright-community/playwright-go"
|
"github.com/playwright-community/playwright-go"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
"go.uber.org/multierr"
|
||||||
|
|
||||||
"oss.terrastruct.com/d2"
|
"oss.terrastruct.com/d2"
|
||||||
"oss.terrastruct.com/d2/d2layouts/d2sequence"
|
"oss.terrastruct.com/d2/d2layouts/d2sequence"
|
||||||
|
|
@ -37,26 +38,26 @@ func run(ctx context.Context, ms *xmain.State) (err error) {
|
||||||
// These should be kept up-to-date with the d2 man page
|
// These should be kept up-to-date with the d2 man page
|
||||||
watchFlag, err := ms.Opts.Bool("D2_WATCH", "watch", "w", false, "watch for changes to input and live reload. Use $HOST and $PORT to specify the listening address.\n(default localhost:0, which is will open on a randomly available local port).")
|
watchFlag, err := ms.Opts.Bool("D2_WATCH", "watch", "w", false, "watch for changes to input and live reload. Use $HOST and $PORT to specify the listening address.\n(default localhost:0, which is will open on a randomly available local port).")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return xmain.UsageErrorf(err.Error())
|
return err
|
||||||
}
|
}
|
||||||
hostFlag := ms.Opts.String("HOST", "host", "h", "localhost", "host listening address when used with watch")
|
hostFlag := ms.Opts.String("HOST", "host", "h", "localhost", "host listening address when used with watch")
|
||||||
portFlag := ms.Opts.String("PORT", "port", "p", "0", "port listening address when used with watch")
|
portFlag := ms.Opts.String("PORT", "port", "p", "0", "port listening address when used with watch")
|
||||||
bundleFlag, err := ms.Opts.Bool("D2_BUNDLE", "bundle", "b", true, "when outputting SVG, bundle all assets and layers into the output file.")
|
bundleFlag, err := ms.Opts.Bool("D2_BUNDLE", "bundle", "b", true, "when outputting SVG, bundle all assets and layers into the output file.")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return xmain.UsageErrorf(err.Error())
|
return err
|
||||||
}
|
}
|
||||||
debugFlag, err := ms.Opts.Bool("DEBUG", "debug", "d", false, "print debug logs.")
|
debugFlag, err := ms.Opts.Bool("DEBUG", "debug", "d", false, "print debug logs.")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return xmain.UsageErrorf(err.Error())
|
return err
|
||||||
}
|
}
|
||||||
layoutFlag := ms.Opts.String("D2_LAYOUT", "layout", "l", "dagre", `the layout engine used.`)
|
layoutFlag := ms.Opts.String("D2_LAYOUT", "layout", "l", "dagre", `the layout engine used.`)
|
||||||
themeFlag, err := ms.Opts.Int64("D2_THEME", "theme", "t", 0, "the diagram theme ID. For a list of available options, see https://oss.terrastruct.com/d2")
|
themeFlag, err := ms.Opts.Int64("D2_THEME", "theme", "t", 0, "the diagram theme ID. For a list of available options, see https://oss.terrastruct.com/d2")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return xmain.UsageErrorf(err.Error())
|
return err
|
||||||
}
|
}
|
||||||
versionFlag, err := ms.Opts.Bool("", "version", "v", false, "get the version")
|
versionFlag, err := ms.Opts.Bool("", "version", "v", false, "get the version")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return xmain.UsageErrorf(err.Error())
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ms.Opts.Flags.Parse(ms.Opts.Args)
|
err = ms.Opts.Flags.Parse(ms.Opts.Args)
|
||||||
|
|
@ -156,6 +157,7 @@ func run(ctx context.Context, ms *xmain.State) (err error) {
|
||||||
port: *portFlag,
|
port: *portFlag,
|
||||||
inputPath: inputPath,
|
inputPath: inputPath,
|
||||||
outputPath: outputPath,
|
outputPath: outputPath,
|
||||||
|
bundle: *bundleFlag,
|
||||||
pw: pw,
|
pw: pw,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -167,27 +169,26 @@ func run(ctx context.Context, ms *xmain.State) (err error) {
|
||||||
ctx, cancel := context.WithTimeout(ctx, time.Minute*2)
|
ctx, cancel := context.WithTimeout(ctx, time.Minute*2)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
if *bundleFlag {
|
_, written, err := compile(ctx, ms, plugin, *themeFlag, inputPath, outputPath, *bundleFlag, pw.Page)
|
||||||
_ = 343
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = compile(ctx, ms, false, plugin, *themeFlag, inputPath, outputPath, pw.Page)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
if written {
|
||||||
|
return fmt.Errorf("failed to fully compile (partial render written): %w", err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("failed to compile: %w", err)
|
||||||
}
|
}
|
||||||
ms.Log.Success.Printf("successfully compiled %v to %v", inputPath, outputPath)
|
ms.Log.Success.Printf("successfully compiled %v to %v", inputPath, outputPath)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, page playwright.Page) ([]byte, error) {
|
func compile(ctx context.Context, ms *xmain.State, plugin d2plugin.Plugin, themeID int64, inputPath, outputPath string, bundle bool, page playwright.Page) (_ []byte, written bool, _ error) {
|
||||||
input, err := ms.ReadPath(inputPath)
|
input, err := ms.ReadPath(inputPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
ruler, err := textmeasure.NewRuler()
|
ruler, err := textmeasure.NewRuler()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
layout := plugin.Layout
|
layout := plugin.Layout
|
||||||
|
|
@ -201,47 +202,46 @@ func compile(ctx context.Context, ms *xmain.State, isWatching bool, plugin d2plu
|
||||||
ThemeID: themeID,
|
ThemeID: themeID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
svg, err := d2svg.Render(d)
|
svg, err := d2svg.Render(d)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
svg, err = plugin.PostProcess(ctx, svg)
|
svg, err = plugin.PostProcess(ctx, svg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return svg, false, err
|
||||||
}
|
}
|
||||||
svg, err = imgbundler.InlineLocal(ctx, ms, svg)
|
|
||||||
if err != nil {
|
svg, bundleErr := imgbundler.BundleLocal(ctx, ms, svg)
|
||||||
ms.Log.Error.Printf("missing/broken local image(s), writing partial output: %v", err)
|
if bundle {
|
||||||
|
var bundleErr2 error
|
||||||
|
svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg)
|
||||||
|
bundleErr = multierr.Combine(bundleErr, bundleErr2)
|
||||||
}
|
}
|
||||||
|
|
||||||
out := svg
|
out := svg
|
||||||
if filepath.Ext(outputPath) == ".png" {
|
if filepath.Ext(outputPath) == ".png" {
|
||||||
svg, err = imgbundler.InlineRemote(ctx, ms, svg)
|
svg := svg
|
||||||
if err != nil {
|
if !bundle {
|
||||||
ms.Log.Error.Printf("missing/broken remote image(s), writing partial output: %v", err)
|
var bundleErr2 error
|
||||||
|
svg, bundleErr2 = imgbundler.BundleRemote(ctx, ms, svg)
|
||||||
|
bundleErr = multierr.Combine(bundleErr, bundleErr2)
|
||||||
}
|
}
|
||||||
|
|
||||||
out, err = png.ConvertSVG(ms, page, svg)
|
out, err = png.ConvertSVG(ms, page, svg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return svg, false, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ms.WritePath(outputPath, out)
|
err = ms.WritePath(outputPath, out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return svg, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Missing/broken images are fine during watch mode, as the user is likely building up a diagram.
|
return svg, true, bundleErr
|
||||||
// Otherwise, the assumption is that this diagram is building for production, and broken images are not okay.
|
|
||||||
if !isWatching && ms.Log.Nerrors() > 0 {
|
|
||||||
return nil, xmain.ExitErrorf(1, "errors logged while rendering, partial output written to %v", outputPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
return svg, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newExt must include leading .
|
// newExt must include leading .
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
#d2c-err {
|
#d2-err {
|
||||||
/* This style was copied from Chrome's svg parser error style. */
|
/* This style was copied from Chrome's svg parser error style. */
|
||||||
white-space: pre-wrap;
|
white-space: pre-wrap;
|
||||||
border: 2px solid #c77;
|
border: 2px solid #c77;
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,9 @@ window.addEventListener("DOMContentLoaded", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
function init(reconnectDelay) {
|
function init(reconnectDelay) {
|
||||||
|
const d2ErrDiv = window.document.querySelector("#d2-err");
|
||||||
|
const d2SVG = window.document.querySelector("#d2-svg");
|
||||||
|
|
||||||
const devMode = document.body.dataset.d2DevMode === "true";
|
const devMode = document.body.dataset.d2DevMode === "true";
|
||||||
const ws = new WebSocket(
|
const ws = new WebSocket(
|
||||||
`ws://${window.location.host}${window.location.pathname}watch`
|
`ws://${window.location.host}${window.location.pathname}watch`
|
||||||
|
|
@ -19,13 +22,7 @@ function init(reconnectDelay) {
|
||||||
} else {
|
} else {
|
||||||
console.debug("watch websocket received data");
|
console.debug("watch websocket received data");
|
||||||
}
|
}
|
||||||
const d2ErrDiv = window.document.querySelector("#d2-err");
|
if (msg.svg) {
|
||||||
if (msg.err) {
|
|
||||||
d2ErrDiv.innerText = msg.err;
|
|
||||||
d2ErrDiv.style.display = "block";
|
|
||||||
d2ErrDiv.scrollIntoView();
|
|
||||||
} else {
|
|
||||||
const d2SVG = window.document.querySelector("#d2-svg");
|
|
||||||
// We could turn d2SVG into an actual SVG element and use outerHTML to fully replace it
|
// We could turn d2SVG into an actual SVG element and use outerHTML to fully replace it
|
||||||
// with the result from the renderer but unfortunately that overwrites the #d2-svg ID.
|
// with the result from the renderer but unfortunately that overwrites the #d2-svg ID.
|
||||||
// Even if you add another line to set it afterwards. The parsing/interpretation of outerHTML must be async.
|
// Even if you add another line to set it afterwards. The parsing/interpretation of outerHTML must be async.
|
||||||
|
|
@ -36,6 +33,11 @@ function init(reconnectDelay) {
|
||||||
d2SVG.innerHTML = msg.svg;
|
d2SVG.innerHTML = msg.svg;
|
||||||
d2ErrDiv.style.display = "none";
|
d2ErrDiv.style.display = "none";
|
||||||
}
|
}
|
||||||
|
if (msg.err) {
|
||||||
|
d2ErrDiv.innerText = msg.err;
|
||||||
|
d2ErrDiv.style.display = "block";
|
||||||
|
d2ErrDiv.scrollIntoView();
|
||||||
|
}
|
||||||
};
|
};
|
||||||
ws.onerror = (ev) => {
|
ws.onerror = (ev) => {
|
||||||
console.error("watch websocket connection error", ev);
|
console.error("watch websocket connection error", ev);
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ type watcherOpts struct {
|
||||||
port string
|
port string
|
||||||
inputPath string
|
inputPath string
|
||||||
outputPath string
|
outputPath string
|
||||||
|
bundle bool
|
||||||
pw png.Playwright
|
pw png.Playwright
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -73,8 +74,8 @@ type watcher struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type compileResult struct {
|
type compileResult struct {
|
||||||
Err string `json:"err"`
|
|
||||||
SVG string `json:"svg"`
|
SVG string `json:"svg"`
|
||||||
|
Err string `json:"err"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func newWatcher(ctx context.Context, ms *xmain.State, opts watcherOpts) (*watcher, error) {
|
func newWatcher(ctx context.Context, ms *xmain.State, opts watcherOpts) (*watcher, error) {
|
||||||
|
|
@ -345,19 +346,23 @@ func (w *watcher) compileLoop(ctx context.Context) error {
|
||||||
w.pw = newPW
|
w.pw = newPW
|
||||||
}
|
}
|
||||||
|
|
||||||
b, err := compile(ctx, w.ms, true, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.pw.Page)
|
svg, _, err := compile(ctx, w.ms, w.layoutPlugin, w.themeID, w.inputPath, w.outputPath, w.bundle, w.pw.Page)
|
||||||
|
errs := ""
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err)
|
if len(svg) > 0 {
|
||||||
w.ms.Log.Error.Print(err)
|
err = fmt.Errorf("failed to fully %scompile (rendering partial svg): %w", recompiledPrefix, err)
|
||||||
w.broadcast(&compileResult{
|
} else {
|
||||||
Err: err.Error(),
|
err = fmt.Errorf("failed to %scompile: %w", recompiledPrefix, err)
|
||||||
})
|
}
|
||||||
|
errs = err.Error()
|
||||||
|
w.ms.Log.Error.Print(errs)
|
||||||
} else {
|
} else {
|
||||||
w.ms.Log.Success.Printf("successfully %scompiled %v to %v", recompiledPrefix, w.inputPath, w.outputPath)
|
w.ms.Log.Success.Printf("successfully %scompiled %v to %v", recompiledPrefix, w.inputPath, w.outputPath)
|
||||||
w.broadcast(&compileResult{
|
|
||||||
SVG: string(b),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
w.broadcast(&compileResult{
|
||||||
|
SVG: string(svg),
|
||||||
|
Err: errs,
|
||||||
|
})
|
||||||
|
|
||||||
if firstCompile {
|
if firstCompile {
|
||||||
firstCompile = false
|
firstCompile = false
|
||||||
|
|
|
||||||
2
go.mod
generated
2
go.mod
generated
|
|
@ -20,6 +20,7 @@ require (
|
||||||
golang.org/x/image v0.1.0
|
golang.org/x/image v0.1.0
|
||||||
golang.org/x/net v0.2.0
|
golang.org/x/net v0.2.0
|
||||||
golang.org/x/text v0.4.0
|
golang.org/x/text v0.4.0
|
||||||
|
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
|
||||||
gonum.org/v1/plot v0.12.0
|
gonum.org/v1/plot v0.12.0
|
||||||
nhooyr.io/websocket v1.8.7
|
nhooyr.io/websocket v1.8.7
|
||||||
oss.terrastruct.com/cmdlog v0.0.0-20221129200109-540ef52ff07d
|
oss.terrastruct.com/cmdlog v0.0.0-20221129200109-540ef52ff07d
|
||||||
|
|
@ -57,7 +58,6 @@ require (
|
||||||
golang.org/x/crypto v0.3.0 // indirect
|
golang.org/x/crypto v0.3.0 // indirect
|
||||||
golang.org/x/sys v0.2.0 // indirect
|
golang.org/x/sys v0.2.0 // indirect
|
||||||
golang.org/x/term v0.2.0 // indirect
|
golang.org/x/term v0.2.0 // indirect
|
||||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
|
||||||
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
|
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
|
||||||
google.golang.org/protobuf v1.28.1 // indirect
|
google.golang.org/protobuf v1.28.1 // indirect
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||||
|
|
|
||||||
|
|
@ -6,15 +6,17 @@ import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"mime"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/multierr"
|
"golang.org/x/xerrors"
|
||||||
"oss.terrastruct.com/xdefer"
|
"oss.terrastruct.com/xdefer"
|
||||||
|
|
||||||
"oss.terrastruct.com/d2/lib/xmain"
|
"oss.terrastruct.com/d2/lib/xmain"
|
||||||
|
|
@ -22,140 +24,186 @@ import (
|
||||||
|
|
||||||
const maxImageSize int64 = 1 << 25 // 33_554_432
|
const maxImageSize int64 = 1 << 25 // 33_554_432
|
||||||
|
|
||||||
var imageRe = regexp.MustCompile(`<image href="([^"]+)"`)
|
var imageRegex = regexp.MustCompile(`<image href="([^"]+)"`)
|
||||||
|
|
||||||
type resp struct {
|
func BundleLocal(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
|
||||||
srctxt string
|
return bundle(ctx, ms, in, false)
|
||||||
data string
|
|
||||||
err error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func InlineLocal(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
|
func BundleRemote(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
|
||||||
return inline(ctx, ms, in, false)
|
return bundle(ctx, ms, in, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func InlineRemote(ctx context.Context, ms *xmain.State, in []byte) ([]byte, error) {
|
type repl struct {
|
||||||
return inline(ctx, ms, in, true)
|
from []byte
|
||||||
|
to []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
func inline(ctx context.Context, ms *xmain.State, svg []byte, isRemote bool) (_ []byte, err error) {
|
func bundle(ctx context.Context, ms *xmain.State, svg []byte, isRemote bool) (_ []byte, err error) {
|
||||||
defer xdefer.Errorf(&err, "failed to bundle images")
|
if isRemote {
|
||||||
imgs := imageRe.FindAllSubmatch(svg, -1)
|
defer xdefer.Errorf(&err, "failed to bundle remote images")
|
||||||
|
} else {
|
||||||
var filtered [][][]byte
|
defer xdefer.Errorf(&err, "failed to bundle local images")
|
||||||
for _, img := range imgs {
|
|
||||||
u, err := url.Parse(string(img[1]))
|
|
||||||
isRemoteImg := err == nil && strings.HasPrefix(u.Scheme, "http")
|
|
||||||
if isRemoteImg == isRemote {
|
|
||||||
filtered = append(filtered, img)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
imgs := imageRegex.FindAllSubmatch(svg, -1)
|
||||||
var wg sync.WaitGroup
|
imgs = filterImageElements(imgs, isRemote)
|
||||||
respChan := make(chan resp)
|
|
||||||
// Limits the number of workers to 16.
|
|
||||||
sema := make(chan struct{}, 16)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
|
ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
wg.Add(len(filtered))
|
return runWorkers(ctx, ms, svg, imgs, isRemote)
|
||||||
|
}
|
||||||
|
|
||||||
|
// filterImageElements finds all image elements in imgs that are eligible
|
||||||
|
// for bundling in the current context.
|
||||||
|
func filterImageElements(imgs [][][]byte, isRemote bool) [][][]byte {
|
||||||
|
imgs2 := imgs[:0]
|
||||||
|
for _, img := range imgs {
|
||||||
|
href := string(img[1])
|
||||||
|
|
||||||
|
// Skip already bundled images.
|
||||||
|
if strings.HasPrefix(href, "data:") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := url.Parse(href)
|
||||||
|
isRemoteImg := err == nil && strings.HasPrefix(u.Scheme, "http")
|
||||||
|
|
||||||
|
if isRemoteImg == isRemote {
|
||||||
|
imgs2 = append(imgs2, img)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return imgs2
|
||||||
|
}
|
||||||
|
|
||||||
|
func runWorkers(ctx context.Context, ms *xmain.State, svg []byte, imgs [][][]byte, isRemote bool) (_ []byte, err error) {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
replc := make(chan repl)
|
||||||
|
|
||||||
|
wg.Add(len(imgs))
|
||||||
|
go func() {
|
||||||
|
wg.Wait()
|
||||||
|
close(replc)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Limits the number of workers to 16.
|
||||||
|
sema := make(chan struct{}, 16)
|
||||||
|
|
||||||
|
var errhrefsMu sync.Mutex
|
||||||
|
var errhrefs []string
|
||||||
|
|
||||||
// Start workers as the sema allows.
|
// Start workers as the sema allows.
|
||||||
go func() {
|
go func() {
|
||||||
for _, img := range filtered {
|
for _, img := range imgs {
|
||||||
|
img := img
|
||||||
sema <- struct{}{}
|
sema <- struct{}{}
|
||||||
go func(src, href string) {
|
go func() {
|
||||||
defer func() {
|
defer func() {
|
||||||
wg.Done()
|
wg.Done()
|
||||||
<-sema
|
<-sema
|
||||||
}()
|
}()
|
||||||
|
|
||||||
var data string
|
bundledImage, err := worker(ctx, img[1], isRemote)
|
||||||
var err error
|
if err != nil {
|
||||||
if isRemote {
|
ms.Log.Error.Printf("failed to bundle %s: %v", img[1], err)
|
||||||
data, err = fetch(ctx, href)
|
errhrefsMu.Lock()
|
||||||
} else {
|
errhrefs = append(errhrefs, string(img[1]))
|
||||||
data, err = read(href)
|
errhrefsMu.Unlock()
|
||||||
|
return
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
case respChan <- resp{
|
case replc <- repl{
|
||||||
srctxt: src,
|
from: img[0],
|
||||||
data: data,
|
to: bundledImage,
|
||||||
err: err,
|
|
||||||
}:
|
}:
|
||||||
}
|
}
|
||||||
}(string(img[0]), string(img[1]))
|
}()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
go func() {
|
t := time.NewTicker(time.Second * 5)
|
||||||
wg.Wait()
|
defer t.Stop()
|
||||||
close(respChan)
|
|
||||||
}()
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil, fmt.Errorf("failed to wait for imgbundler workers: %w", ctx.Err())
|
return svg, xerrors.Errorf("failed to wait for workers: %w", ctx.Err())
|
||||||
case <-time.After(time.Second * 5):
|
case <-t.C:
|
||||||
ms.Log.Info.Printf("fetching images...")
|
ms.Log.Info.Printf("fetching images...")
|
||||||
case resp, ok := <-respChan:
|
case repl, ok := <-replc:
|
||||||
if !ok {
|
if !ok {
|
||||||
return svg, err
|
if len(errhrefs) > 0 {
|
||||||
|
return svg, xerrors.Errorf("%v", errhrefs)
|
||||||
|
}
|
||||||
|
return svg, nil
|
||||||
}
|
}
|
||||||
if resp.err != nil {
|
svg = bytes.Replace(svg, repl.from, repl.to, 1)
|
||||||
err = multierr.Combine(err, resp.err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
svg = bytes.Replace(svg, []byte(resp.srctxt), []byte(fmt.Sprintf(`<image href="%s"`, resp.data)), 1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var imgClient = &http.Client{}
|
func worker(ctx context.Context, href []byte, isRemote bool) ([]byte, error) {
|
||||||
|
var buf []byte
|
||||||
|
var mimeType string
|
||||||
|
var err error
|
||||||
|
if isRemote {
|
||||||
|
buf, mimeType, err = httpGet(ctx, string(href))
|
||||||
|
} else {
|
||||||
|
buf, err = os.ReadFile(string(href))
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
func fetch(ctx context.Context, href string) (string, error) {
|
if mimeType == "" {
|
||||||
|
mimeType = sniffMimeType(href, buf, isRemote)
|
||||||
|
}
|
||||||
|
mimeType = strings.Replace(mimeType, "text/xml", "image/svg+xml", 1)
|
||||||
|
b64 := base64.StdEncoding.EncodeToString(buf)
|
||||||
|
return []byte(fmt.Sprintf(`<image href="data:%s;base64,%s"`, mimeType, b64)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var httpClient = &http.Client{}
|
||||||
|
|
||||||
|
func httpGet(ctx context.Context, href string) ([]byte, string, error) {
|
||||||
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", href, nil)
|
req, err := http.NewRequestWithContext(ctx, "GET", href, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
imgResp, err := imgClient.Do(req)
|
resp, err := httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
defer imgResp.Body.Close()
|
defer resp.Body.Close()
|
||||||
if imgResp.StatusCode != 200 {
|
if resp.StatusCode != 200 {
|
||||||
return "", fmt.Errorf("img %s returned status code %d", href, imgResp.StatusCode)
|
return nil, "", fmt.Errorf("expected status 200 but got %d %s", resp.StatusCode, resp.Status)
|
||||||
}
|
}
|
||||||
r := http.MaxBytesReader(nil, imgResp.Body, maxImageSize)
|
r := http.MaxBytesReader(nil, resp.Body, maxImageSize)
|
||||||
data, err := ioutil.ReadAll(r)
|
buf, err := ioutil.ReadAll(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
|
return buf, resp.Header.Get("Content-Type"), nil
|
||||||
mimeType := http.DetectContentType(data)
|
|
||||||
mimeType = strings.Replace(mimeType, "text/xml", "image/svg+xml", 1)
|
|
||||||
|
|
||||||
enc := base64.StdEncoding.EncodeToString(data)
|
|
||||||
|
|
||||||
return fmt.Sprintf("data:%s;base64,%s", mimeType, enc), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func read(href string) (string, error) {
|
// sniffMimeType sniffs the mime type of href based on its file extension and contents.
|
||||||
data, err := os.ReadFile(href)
|
func sniffMimeType(href, buf []byte, isRemote bool) string {
|
||||||
if err != nil {
|
p := string(href)
|
||||||
return "", err
|
if isRemote {
|
||||||
|
u, err := url.Parse(p)
|
||||||
|
if err != nil {
|
||||||
|
p = ""
|
||||||
|
} else {
|
||||||
|
p = u.Path
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
mimeType := mime.TypeByExtension(path.Ext(p))
|
||||||
mimeType := http.DetectContentType(data)
|
if mimeType == "" {
|
||||||
mimeType = strings.Replace(mimeType, "text/xml", "image/svg+xml", 1)
|
mimeType = http.DetectContentType(buf)
|
||||||
|
}
|
||||||
enc := base64.StdEncoding.EncodeToString(data)
|
return mimeType
|
||||||
|
|
||||||
return fmt.Sprintf("data:%s;base64,%s", mimeType, enc), nil
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ func TestRegex(t *testing.T) {
|
||||||
|
|
||||||
for _, href := range append(urls, notURLs...) {
|
for _, href := range append(urls, notURLs...) {
|
||||||
str := fmt.Sprintf(`<image href="%s" />`, href)
|
str := fmt.Sprintf(`<image href="%s" />`, href)
|
||||||
matches := imageRe.FindAllStringSubmatch(str, -1)
|
matches := imageRegex.FindAllStringSubmatch(str, -1)
|
||||||
if len(matches) != 1 {
|
if len(matches) != 1 {
|
||||||
t.Fatalf("uri regex didn't match %s", str)
|
t.Fatalf("uri regex didn't match %s", str)
|
||||||
}
|
}
|
||||||
|
|
@ -90,7 +90,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
|
||||||
}
|
}
|
||||||
ms.Log = cmdlog.Log(ms.Env, os.Stderr)
|
ms.Log = cmdlog.Log(ms.Env, os.Stderr)
|
||||||
|
|
||||||
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
||||||
respRecorder := httptest.NewRecorder()
|
respRecorder := httptest.NewRecorder()
|
||||||
switch req.URL.String() {
|
switch req.URL.String() {
|
||||||
case svgURL:
|
case svgURL:
|
||||||
|
|
@ -104,7 +104,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
|
||||||
return respRecorder.Result()
|
return respRecorder.Result()
|
||||||
})
|
})
|
||||||
|
|
||||||
out, err := InlineRemote(ctx, ms, []byte(sampleSVG))
|
out, err := BundleRemote(ctx, ms, []byte(sampleSVG))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
@ -119,7 +119,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test almost too large response
|
// Test almost too large response
|
||||||
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
||||||
respRecorder := httptest.NewRecorder()
|
respRecorder := httptest.NewRecorder()
|
||||||
bytes := make([]byte, maxImageSize)
|
bytes := make([]byte, maxImageSize)
|
||||||
rand.Read(bytes)
|
rand.Read(bytes)
|
||||||
|
|
@ -127,13 +127,13 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
|
||||||
respRecorder.WriteHeader(200)
|
respRecorder.WriteHeader(200)
|
||||||
return respRecorder.Result()
|
return respRecorder.Result()
|
||||||
})
|
})
|
||||||
_, err = InlineRemote(ctx, ms, []byte(sampleSVG))
|
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test too large response
|
// Test too large response
|
||||||
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
||||||
respRecorder := httptest.NewRecorder()
|
respRecorder := httptest.NewRecorder()
|
||||||
bytes := make([]byte, maxImageSize+1)
|
bytes := make([]byte, maxImageSize+1)
|
||||||
rand.Read(bytes)
|
rand.Read(bytes)
|
||||||
|
|
@ -141,18 +141,18 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
|
||||||
respRecorder.WriteHeader(200)
|
respRecorder.WriteHeader(200)
|
||||||
return respRecorder.Result()
|
return respRecorder.Result()
|
||||||
})
|
})
|
||||||
_, err = InlineRemote(ctx, ms, []byte(sampleSVG))
|
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error")
|
t.Fatal("expected error")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test error response
|
// Test error response
|
||||||
imgClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
httpClient.Transport = roundTripFunc(func(req *http.Request) *http.Response {
|
||||||
respRecorder := httptest.NewRecorder()
|
respRecorder := httptest.NewRecorder()
|
||||||
respRecorder.WriteHeader(500)
|
respRecorder.WriteHeader(500)
|
||||||
return respRecorder.Result()
|
return respRecorder.Result()
|
||||||
})
|
})
|
||||||
_, err = InlineRemote(ctx, ms, []byte(sampleSVG))
|
_, err = BundleRemote(ctx, ms, []byte(sampleSVG))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error")
|
t.Fatal("expected error")
|
||||||
}
|
}
|
||||||
|
|
@ -205,7 +205,7 @@ width="328" height="587" viewBox="-100 -131 328 587"><style type="text/css">
|
||||||
Env: xos.NewEnv(os.Environ()),
|
Env: xos.NewEnv(os.Environ()),
|
||||||
}
|
}
|
||||||
ms.Log = cmdlog.Log(ms.Env, os.Stderr)
|
ms.Log = cmdlog.Log(ms.Env, os.Stderr)
|
||||||
out, err := InlineLocal(ctx, ms, []byte(sampleSVG))
|
out, err := BundleLocal(ctx, ms, []byte(sampleSVG))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -129,7 +129,7 @@ func (o *Opts) Int64(envKey, flag, shortFlag string, defaultVal int64, usage str
|
||||||
if env := o.getEnv(flag, envKey); env != "" {
|
if env := o.getEnv(flag, envKey); env != "" {
|
||||||
envVal, err := strconv.ParseInt(env, 10, 64)
|
envVal, err := strconv.ParseInt(env, 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf(`invalid environment variable %s. Expected int64. Found "%v".`, envKey, envVal)
|
return nil, UsageErrorf(`invalid environment variable %s. Expected int64. Found "%v".`, envKey, envVal)
|
||||||
}
|
}
|
||||||
defaultVal = envVal
|
defaultVal = envVal
|
||||||
}
|
}
|
||||||
|
|
@ -148,7 +148,7 @@ func (o *Opts) String(envKey, flag, shortFlag string, defaultVal, usage string)
|
||||||
func (o *Opts) Bool(envKey, flag, shortFlag string, defaultVal bool, usage string) (*bool, error) {
|
func (o *Opts) Bool(envKey, flag, shortFlag string, defaultVal bool, usage string) (*bool, error) {
|
||||||
if env := o.getEnv(flag, envKey); env != "" {
|
if env := o.getEnv(flag, envKey); env != "" {
|
||||||
if !boolyEnv(env) {
|
if !boolyEnv(env) {
|
||||||
return nil, fmt.Errorf(`invalid environment variable %s. Expected bool. Found "%s".`, envKey, env)
|
return nil, UsageErrorf(`invalid environment variable %s. Expected bool. Found "%s".`, envKey, env)
|
||||||
}
|
}
|
||||||
if truthyEnv(env) {
|
if truthyEnv(env) {
|
||||||
defaultVal = true
|
defaultVal = true
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue