From 0314a74eab1007d9089e786aaa081a41f4778af8 Mon Sep 17 00:00:00 2001
From: Alexander Wang <alex@terrastruct.com>
Date: Sat, 14 Jan 2023 14:20:08 -0800
Subject: [PATCH 1/3] unescape img hrefs

---
 .../query_param_escape/dagre/board.exp.json   | 51 +++++++++++++++++++
 .../query_param_escape/dagre/sketch.exp.svg   |  6 +--
 .../query_param_escape/elk/board.exp.json     | 51 +++++++++++++++++++
 .../query_param_escape/elk/sketch.exp.svg     |  6 +--
 lib/imgbundler/imgbundler.go                  |  9 ++--
 5 files changed, 113 insertions(+), 10 deletions(-)

diff --git a/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json b/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json
index fb0febb97..cd9d2e542 100644
--- a/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json
+++ b/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json
@@ -52,6 +52,57 @@
       "labelPosition": "INSIDE_TOP_CENTER",
       "zIndex": 0,
       "level": 1
+    },
+    {
+      "id": "with ampersand",
+      "type": "",
+      "pos": {
+        "x": 250,
+        "y": 0
+      },
+      "width": 220,
+      "height": 152,
+      "opacity": 1,
+      "strokeDash": 0,
+      "strokeWidth": 2,
+      "borderRadius": 0,
+      "fill": "#F7F8FE",
+      "stroke": "#0D32B2",
+      "shadow": false,
+      "3d": false,
+      "multiple": false,
+      "tooltip": "",
+      "link": "",
+      "icon": {
+        "Scheme": "https",
+        "Opaque": "",
+        "User": null,
+        "Host": "icons.terrastruct.com",
+        "Path": "/aws/Networking & Content Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg",
+        "RawPath": "",
+        "ForceQuery": false,
+        "RawQuery": "",
+        "Fragment": "",
+        "RawFragment": ""
+      },
+      "iconPosition": "INSIDE_MIDDLE_CENTER",
+      "blend": false,
+      "fields": null,
+      "methods": null,
+      "columns": null,
+      "label": "with ampersand",
+      "fontSize": 16,
+      "fontFamily": "DEFAULT",
+      "language": "",
+      "color": "#0A0F25",
+      "italic": false,
+      "bold": true,
+      "underline": false,
+      "labelWidth": 120,
+      "labelHeight": 26,
+      "labelPosition": "INSIDE_TOP_CENTER",
+      "zIndex": 0,
+      "level": 1
     }
   ],
   "connections": []
diff --git a/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg b/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg
index 70f3449d4..8d42301fe 100644
--- a/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg
+++ b/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg
@@ -3,7 +3,7 @@
 id="d2-svg"
 style="background: white;"
 xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-width="394" height="356" viewBox="-102 -102 394 356"><style type="text/css">
+width="674" height="356" viewBox="-102 -102 674 356"><style type="text/css">
 <![CDATA[
 .shape {
   shape-rendering: geometricPrecision;
@@ -39,8 +39,8 @@ width="394" height="356" viewBox="-102 -102 394 356"><style type="text/css">
     svgEl.setAttribute("height", height * ratio - 16);
   }
 });
-]]></script><g id="my network"><g class="shape" ><rect x="0" y="0" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="63.000000" y="44.000000" width="64" height="64" /><text class="text-bold" x="95.000000" y="21.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><mask id="3050687478" maskUnits="userSpaceOnUse" x="-100" y="-100" width="394" height="356">
-<rect x="-100" y="-100" width="394" height="356" fill="white"></rect>
+]]></script><g id="my network"><g class="shape" ><rect x="0" y="0" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="63.000000" y="44.000000" width="64" height="64" /><text class="text-bold" x="95.000000" y="21.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><g id="with ampersand"><g class="shape" ><rect x="250" y="0" width="220" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/aws/Networking%20&amp;%20Content%20Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg" x="328.000000" y="44.000000" width="64" height="64" /><text class="text-bold" x="360.000000" y="21.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">with ampersand</text></g><mask id="1054261257" maskUnits="userSpaceOnUse" x="-100" y="-100" width="674" height="356">
+<rect x="-100" y="-100" width="674" height="356" fill="white"></rect>
 
 </mask><style type="text/css"><![CDATA[
 .text-bold {
diff --git a/e2etests/testdata/regression/query_param_escape/elk/board.exp.json b/e2etests/testdata/regression/query_param_escape/elk/board.exp.json
index bce062b6f..8c5e93270 100644
--- a/e2etests/testdata/regression/query_param_escape/elk/board.exp.json
+++ b/e2etests/testdata/regression/query_param_escape/elk/board.exp.json
@@ -52,6 +52,57 @@
       "labelPosition": "INSIDE_TOP_CENTER",
       "zIndex": 0,
       "level": 1
+    },
+    {
+      "id": "with ampersand",
+      "type": "",
+      "pos": {
+        "x": 222,
+        "y": 12
+      },
+      "width": 220,
+      "height": 152,
+      "opacity": 1,
+      "strokeDash": 0,
+      "strokeWidth": 2,
+      "borderRadius": 0,
+      "fill": "#F7F8FE",
+      "stroke": "#0D32B2",
+      "shadow": false,
+      "3d": false,
+      "multiple": false,
+      "tooltip": "",
+      "link": "",
+      "icon": {
+        "Scheme": "https",
+        "Opaque": "",
+        "User": null,
+        "Host": "icons.terrastruct.com",
+        "Path": "/aws/Networking & Content Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg",
+        "RawPath": "",
+        "ForceQuery": false,
+        "RawQuery": "",
+        "Fragment": "",
+        "RawFragment": ""
+      },
+      "iconPosition": "INSIDE_MIDDLE_CENTER",
+      "blend": false,
+      "fields": null,
+      "methods": null,
+      "columns": null,
+      "label": "with ampersand",
+      "fontSize": 16,
+      "fontFamily": "DEFAULT",
+      "language": "",
+      "color": "#0A0F25",
+      "italic": false,
+      "bold": true,
+      "underline": false,
+      "labelWidth": 120,
+      "labelHeight": 26,
+      "labelPosition": "INSIDE_TOP_CENTER",
+      "zIndex": 0,
+      "level": 1
     }
   ],
   "connections": []
diff --git a/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg b/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg
index c0072a72c..841b16998 100644
--- a/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg
+++ b/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg
@@ -3,7 +3,7 @@
 id="d2-svg"
 style="background: white;"
 xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-width="394" height="356" viewBox="-90 -90 394 356"><style type="text/css">
+width="634" height="356" viewBox="-90 -90 634 356"><style type="text/css">
 <![CDATA[
 .shape {
   shape-rendering: geometricPrecision;
@@ -39,8 +39,8 @@ width="394" height="356" viewBox="-90 -90 394 356"><style type="text/css">
     svgEl.setAttribute("height", height * ratio - 16);
   }
 });
-]]></script><g id="my network"><g class="shape" ><rect x="12" y="12" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="75.000000" y="56.000000" width="64" height="64" /><text class="text-bold" x="107.000000" y="33.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><mask id="2671818094" maskUnits="userSpaceOnUse" x="-100" y="-100" width="394" height="356">
-<rect x="-100" y="-100" width="394" height="356" fill="white"></rect>
+]]></script><g id="my network"><g class="shape" ><rect x="12" y="12" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="75.000000" y="56.000000" width="64" height="64" /><text class="text-bold" x="107.000000" y="33.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><g id="with ampersand"><g class="shape" ><rect x="222" y="12" width="220" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/aws/Networking%20&amp;%20Content%20Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg" x="300.000000" y="56.000000" width="64" height="64" /><text class="text-bold" x="332.000000" y="33.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">with ampersand</text></g><mask id="2491478855" maskUnits="userSpaceOnUse" x="-100" y="-100" width="634" height="356">
+<rect x="-100" y="-100" width="634" height="356" fill="white"></rect>
 
 </mask><style type="text/css"><![CDATA[
 .text-bold {
diff --git a/lib/imgbundler/imgbundler.go b/lib/imgbundler/imgbundler.go
index e1b719687..5c6de7038 100644
--- a/lib/imgbundler/imgbundler.go
+++ b/lib/imgbundler/imgbundler.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"html"
 	"io/ioutil"
 	"mime"
 	"net/http"
@@ -66,7 +67,7 @@ func filterImageElements(imgs [][][]byte, isRemote bool) [][][]byte {
 			continue
 		}
 
-		u, err := url.Parse(href)
+		u, err := url.Parse(html.UnescapeString(href))
 		isRemoteImg := err == nil && strings.HasPrefix(u.Scheme, "http")
 
 		if isRemoteImg == isRemote {
@@ -147,9 +148,9 @@ func worker(ctx context.Context, href []byte, isRemote bool) ([]byte, error) {
 	var mimeType string
 	var err error
 	if isRemote {
-		buf, mimeType, err = httpGet(ctx, string(href))
+		buf, mimeType, err = httpGet(ctx, html.UnescapeString(string(href)))
 	} else {
-		buf, err = os.ReadFile(string(href))
+		buf, err = os.ReadFile(html.UnescapeString(string(href)))
 	}
 	if err != nil {
 		return nil, err
@@ -194,7 +195,7 @@ func httpGet(ctx context.Context, href string) ([]byte, string, error) {
 func sniffMimeType(href, buf []byte, isRemote bool) string {
 	p := string(href)
 	if isRemote {
-		u, err := url.Parse(p)
+		u, err := url.Parse(html.UnescapeString(p))
 		if err != nil {
 			p = ""
 		} else {

From 68f131672a1d07f5a58d7a31e0d9314de95a7fa8 Mon Sep 17 00:00:00 2001
From: Alexander Wang <alex@terrastruct.com>
Date: Sat, 14 Jan 2023 14:21:04 -0800
Subject: [PATCH 2/3] revert test

---
 .../query_param_escape/dagre/board.exp.json   | 51 -------------------
 .../query_param_escape/dagre/sketch.exp.svg   |  6 +--
 .../query_param_escape/elk/board.exp.json     | 51 -------------------
 .../query_param_escape/elk/sketch.exp.svg     |  6 +--
 4 files changed, 6 insertions(+), 108 deletions(-)

diff --git a/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json b/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json
index cd9d2e542..fb0febb97 100644
--- a/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json
+++ b/e2etests/testdata/regression/query_param_escape/dagre/board.exp.json
@@ -52,57 +52,6 @@
       "labelPosition": "INSIDE_TOP_CENTER",
       "zIndex": 0,
       "level": 1
-    },
-    {
-      "id": "with ampersand",
-      "type": "",
-      "pos": {
-        "x": 250,
-        "y": 0
-      },
-      "width": 220,
-      "height": 152,
-      "opacity": 1,
-      "strokeDash": 0,
-      "strokeWidth": 2,
-      "borderRadius": 0,
-      "fill": "#F7F8FE",
-      "stroke": "#0D32B2",
-      "shadow": false,
-      "3d": false,
-      "multiple": false,
-      "tooltip": "",
-      "link": "",
-      "icon": {
-        "Scheme": "https",
-        "Opaque": "",
-        "User": null,
-        "Host": "icons.terrastruct.com",
-        "Path": "/aws/Networking & Content Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg",
-        "RawPath": "",
-        "ForceQuery": false,
-        "RawQuery": "",
-        "Fragment": "",
-        "RawFragment": ""
-      },
-      "iconPosition": "INSIDE_MIDDLE_CENTER",
-      "blend": false,
-      "fields": null,
-      "methods": null,
-      "columns": null,
-      "label": "with ampersand",
-      "fontSize": 16,
-      "fontFamily": "DEFAULT",
-      "language": "",
-      "color": "#0A0F25",
-      "italic": false,
-      "bold": true,
-      "underline": false,
-      "labelWidth": 120,
-      "labelHeight": 26,
-      "labelPosition": "INSIDE_TOP_CENTER",
-      "zIndex": 0,
-      "level": 1
     }
   ],
   "connections": []
diff --git a/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg b/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg
index 8d42301fe..70f3449d4 100644
--- a/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg
+++ b/e2etests/testdata/regression/query_param_escape/dagre/sketch.exp.svg
@@ -3,7 +3,7 @@
 id="d2-svg"
 style="background: white;"
 xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-width="674" height="356" viewBox="-102 -102 674 356"><style type="text/css">
+width="394" height="356" viewBox="-102 -102 394 356"><style type="text/css">
 <![CDATA[
 .shape {
   shape-rendering: geometricPrecision;
@@ -39,8 +39,8 @@ width="674" height="356" viewBox="-102 -102 674 356"><style type="text/css">
     svgEl.setAttribute("height", height * ratio - 16);
   }
 });
-]]></script><g id="my network"><g class="shape" ><rect x="0" y="0" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="63.000000" y="44.000000" width="64" height="64" /><text class="text-bold" x="95.000000" y="21.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><g id="with ampersand"><g class="shape" ><rect x="250" y="0" width="220" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/aws/Networking%20&amp;%20Content%20Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg" x="328.000000" y="44.000000" width="64" height="64" /><text class="text-bold" x="360.000000" y="21.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">with ampersand</text></g><mask id="1054261257" maskUnits="userSpaceOnUse" x="-100" y="-100" width="674" height="356">
-<rect x="-100" y="-100" width="674" height="356" fill="white"></rect>
+]]></script><g id="my network"><g class="shape" ><rect x="0" y="0" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="63.000000" y="44.000000" width="64" height="64" /><text class="text-bold" x="95.000000" y="21.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><mask id="3050687478" maskUnits="userSpaceOnUse" x="-100" y="-100" width="394" height="356">
+<rect x="-100" y="-100" width="394" height="356" fill="white"></rect>
 
 </mask><style type="text/css"><![CDATA[
 .text-bold {
diff --git a/e2etests/testdata/regression/query_param_escape/elk/board.exp.json b/e2etests/testdata/regression/query_param_escape/elk/board.exp.json
index 8c5e93270..bce062b6f 100644
--- a/e2etests/testdata/regression/query_param_escape/elk/board.exp.json
+++ b/e2etests/testdata/regression/query_param_escape/elk/board.exp.json
@@ -52,57 +52,6 @@
       "labelPosition": "INSIDE_TOP_CENTER",
       "zIndex": 0,
       "level": 1
-    },
-    {
-      "id": "with ampersand",
-      "type": "",
-      "pos": {
-        "x": 222,
-        "y": 12
-      },
-      "width": 220,
-      "height": 152,
-      "opacity": 1,
-      "strokeDash": 0,
-      "strokeWidth": 2,
-      "borderRadius": 0,
-      "fill": "#F7F8FE",
-      "stroke": "#0D32B2",
-      "shadow": false,
-      "3d": false,
-      "multiple": false,
-      "tooltip": "",
-      "link": "",
-      "icon": {
-        "Scheme": "https",
-        "Opaque": "",
-        "User": null,
-        "Host": "icons.terrastruct.com",
-        "Path": "/aws/Networking & Content Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg",
-        "RawPath": "",
-        "ForceQuery": false,
-        "RawQuery": "",
-        "Fragment": "",
-        "RawFragment": ""
-      },
-      "iconPosition": "INSIDE_MIDDLE_CENTER",
-      "blend": false,
-      "fields": null,
-      "methods": null,
-      "columns": null,
-      "label": "with ampersand",
-      "fontSize": 16,
-      "fontFamily": "DEFAULT",
-      "language": "",
-      "color": "#0A0F25",
-      "italic": false,
-      "bold": true,
-      "underline": false,
-      "labelWidth": 120,
-      "labelHeight": 26,
-      "labelPosition": "INSIDE_TOP_CENTER",
-      "zIndex": 0,
-      "level": 1
     }
   ],
   "connections": []
diff --git a/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg b/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg
index 841b16998..c0072a72c 100644
--- a/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg
+++ b/e2etests/testdata/regression/query_param_escape/elk/sketch.exp.svg
@@ -3,7 +3,7 @@
 id="d2-svg"
 style="background: white;"
 xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
-width="634" height="356" viewBox="-90 -90 634 356"><style type="text/css">
+width="394" height="356" viewBox="-90 -90 394 356"><style type="text/css">
 <![CDATA[
 .shape {
   shape-rendering: geometricPrecision;
@@ -39,8 +39,8 @@ width="634" height="356" viewBox="-90 -90 634 356"><style type="text/css">
     svgEl.setAttribute("height", height * ratio - 16);
   }
 });
-]]></script><g id="my network"><g class="shape" ><rect x="12" y="12" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="75.000000" y="56.000000" width="64" height="64" /><text class="text-bold" x="107.000000" y="33.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><g id="with ampersand"><g class="shape" ><rect x="222" y="12" width="220" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/aws/Networking%20&amp;%20Content%20Delivery/Amazon-API-Gateway_Endpoint_light-bg.svg" x="300.000000" y="56.000000" width="64" height="64" /><text class="text-bold" x="332.000000" y="33.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">with ampersand</text></g><mask id="2491478855" maskUnits="userSpaceOnUse" x="-100" y="-100" width="634" height="356">
-<rect x="-100" y="-100" width="634" height="356" fill="white"></rect>
+]]></script><g id="my network"><g class="shape" ><rect x="12" y="12" width="190" height="152" style="fill:#F7F8FE;stroke:#0D32B2;opacity:1.000000;stroke-width:2;" /></g><image href="https://icons.terrastruct.com/infra/019-network.svg?fuga=1&amp;hoge" x="75.000000" y="56.000000" width="64" height="64" /><text class="text-bold" x="107.000000" y="33.000000" style="text-anchor:middle;font-size:16px;fill:#0A0F25">my network</text></g><mask id="2671818094" maskUnits="userSpaceOnUse" x="-100" y="-100" width="394" height="356">
+<rect x="-100" y="-100" width="394" height="356" fill="white"></rect>
 
 </mask><style type="text/css"><![CDATA[
 .text-bold {

From 67cbd568856309bb74bace4dce3ce6da1c93c9a2 Mon Sep 17 00:00:00 2001
From: Alexander Wang <alex@terrastruct.com>
Date: Sat, 14 Jan 2023 14:23:17 -0800
Subject: [PATCH 3/3] changelog

---
 ci/release/changelogs/next.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/release/changelogs/next.md b/ci/release/changelogs/next.md
index d94bb45fe..c3fb0f722 100644
--- a/ci/release/changelogs/next.md
+++ b/ci/release/changelogs/next.md
@@ -15,3 +15,4 @@
 - Fixes tooltip/link attributes being ignored for `sql_table` and `class`. [#658](https://github.com/terrastruct/d2/pull/658)
 - Fixes arrowheads sometimes appearing broken with sketch on. [#656](https://github.com/terrastruct/d2/pull/656)
 - Fixes code snippets not being tall enough with leading newlines. [#664](https://github.com/terrastruct/d2/pull/664)
+- Icon URLs that needed escaping (e.g. with ampersands) are handled correctly by CLI. [#666](https://github.com/terrastruct/d2/pull/666)