From e7f4df37ada684dd07fc2867dfeeba62d51389b6 Mon Sep 17 00:00:00 2001 From: Alexander Wang Date: Sun, 12 Jan 2025 12:08:17 -0700 Subject: [PATCH 1/3] separate node esm and browser esm builds --- d2js/js/.gitignore | 1 + d2js/js/build.js | 121 ++++++++++++++++++++++----- d2js/js/package.json | 15 ++-- d2js/js/src/index.js | 2 +- d2js/js/src/platform.browser.js | 29 +++++++ d2js/js/src/platform.js | 38 +-------- d2js/js/src/platform.node.js | 40 +++++++++ d2js/js/src/wasm-loader.node.js | 8 ++ d2js/js/src/worker.js | 2 - d2js/js/test/integration/cjs.test.js | 2 +- d2js/js/test/integration/esm.test.js | 2 +- d2js/js/test/unit/basic.test.js | 2 +- 12 files changed, 188 insertions(+), 74 deletions(-) create mode 100644 d2js/js/src/platform.browser.js create mode 100644 d2js/js/src/platform.node.js create mode 100644 d2js/js/src/wasm-loader.node.js diff --git a/d2js/js/.gitignore b/d2js/js/.gitignore index 8c8dfb799..3d458b49a 100644 --- a/d2js/js/.gitignore +++ b/d2js/js/.gitignore @@ -2,6 +2,7 @@ node_modules .npm bun.lockb +src/wasm-loader.browser.js wasm/d2.wasm dist/ diff --git a/d2js/js/build.js b/d2js/js/build.js index 403c02d7c..c7a1fcca5 100644 --- a/d2js/js/build.js +++ b/d2js/js/build.js @@ -1,35 +1,110 @@ import { build } from "bun"; -import { copyFile, mkdir } from "node:fs/promises"; -import { join } from "node:path"; +import { copyFile, mkdir, writeFile, readFile, rm } from "node:fs/promises"; +import { join, resolve } from "node:path"; -await mkdir("./dist/esm", { recursive: true }); -await mkdir("./dist/cjs", { recursive: true }); +const __dirname = new URL(".", import.meta.url).pathname; +const ROOT_DIR = resolve(__dirname); +const SRC_DIR = resolve(ROOT_DIR, "src"); + +await rm("./dist", { recursive: true, force: true }); +await mkdir("./dist/browser", { recursive: true }); +await mkdir("./dist/node-esm", { recursive: true }); +await mkdir("./dist/node-cjs", { recursive: true }); + +const wasmBinary = await readFile("./wasm/d2.wasm"); +const wasmExecJs = await readFile("./wasm/wasm_exec.js", "utf8"); + +await writeFile( + join(SRC_DIR, "wasm-loader.browser.js"), + `export const wasmBinary = Uint8Array.from(atob("${Buffer.from(wasmBinary).toString( + "base64" + )}"), c => c.charCodeAt(0)); + export const wasmExecJs = ${JSON.stringify(wasmExecJs)};` +); const commonConfig = { - target: "node", splitting: false, sourcemap: "external", minify: true, - naming: { - entry: "[dir]/[name].js", - chunk: "[name]-[hash].js", - asset: "[name]-[hash][ext]", - }, }; -async function buildAndCopy(format) { - const outdir = `./dist/${format}`; +async function buildPlatformFile(platform) { + const platformContent = + platform === "node" + ? "export * from './platform.node.js';" + : "export * from './platform.browser.js';"; - await build({ - ...commonConfig, - entrypoints: ["./src/index.js", "./src/worker.js", "./src/platform.js"], - outdir, - format, - }); - - await copyFile("./wasm/d2.wasm", join(outdir, "d2.wasm")); - await copyFile("./wasm/wasm_exec.js", join(outdir, "wasm_exec.js")); + const platformPath = join(SRC_DIR, "platform.js"); + await writeFile(platformPath, platformContent); } -await buildAndCopy("esm"); -await buildAndCopy("cjs"); +async function buildAndCopy(buildType) { + const configs = { + browser: { + outdir: resolve(ROOT_DIR, "dist/browser"), + format: "esm", + target: "browser", + platform: "browser", + loader: { + ".js": "jsx", + }, + entrypoints: [ + resolve(SRC_DIR, "index.js"), + resolve(SRC_DIR, "worker.js"), + resolve(SRC_DIR, "platform.js"), + resolve(SRC_DIR, "wasm-loader.browser.js"), + ], + }, + "node-esm": { + outdir: resolve(ROOT_DIR, "dist/node-esm"), + format: "esm", + target: "node", + platform: "node", + entrypoints: [ + resolve(SRC_DIR, "index.js"), + resolve(SRC_DIR, "worker.js"), + resolve(SRC_DIR, "platform.js"), + ], + }, + "node-cjs": { + outdir: resolve(ROOT_DIR, "dist/node-cjs"), + format: "cjs", + target: "node", + platform: "node", + entrypoints: [ + resolve(SRC_DIR, "index.js"), + resolve(SRC_DIR, "worker.js"), + resolve(SRC_DIR, "platform.js"), + ], + }, + }; + + const config = configs[buildType]; + await buildPlatformFile(config.platform); + + const result = await build({ + ...commonConfig, + ...config, + }); + + if (!result.outputs || result.outputs.length === 0) { + throw new Error(`No outputs generated for ${buildType} build`); + } + + if (buildType !== "browser") { + await copyFile(resolve(ROOT_DIR, "wasm/d2.wasm"), join(config.outdir, "d2.wasm")); + await copyFile( + resolve(ROOT_DIR, "wasm/wasm_exec.js"), + join(config.outdir, "wasm_exec.js") + ); + } +} + +try { + await buildAndCopy("browser"); + await buildAndCopy("node-esm"); + await buildAndCopy("node-cjs"); +} catch (error) { + console.error("Build failed:", error); + process.exit(1); +} diff --git a/d2js/js/package.json b/d2js/js/package.json index bfe5720a4..8e56af2e8 100644 --- a/d2js/js/package.json +++ b/d2js/js/package.json @@ -2,10 +2,10 @@ "name": "@terrastruct/d2", "author": "Terrastruct, Inc.", "description": "D2.js is a wrapper around the WASM build of D2, the modern text-to-diagram language.", - "version": "0.1.0", + "version": "0.1.11", "repository": { "type": "git", - "url": "https://github.com/terrastruct/d2.git", + "url": "git+https://github.com/terrastruct/d2.git", "directory": "d2js/js" }, "bugs": { @@ -20,8 +20,10 @@ "module": "./dist/esm/index.js", "exports": { ".": { - "import": "./dist/esm/index.js", - "require": "./dist/cjs/index.js" + "browser": "./dist/browser/index.js", + "import": "./dist/node-esm/index.js", + "require": "./dist/node-cjs/index.js", + "default": "./dist/node-esm/index.js" } }, "files": [ @@ -33,7 +35,7 @@ "test:integration": "bun test test/integration", "test:all": "bun run test && bun run test:integration", "dev": "bun --watch dev-server.js", - "prepublishOnly": "./make.sh" + "prepublishOnly": "./make.sh all" }, "keywords": [ "d2", @@ -43,9 +45,6 @@ "text-to-diagram", "go" ], - "engines": { - "bun": ">=1.0.0" - }, "license": "MPL-2.0", "devDependencies": { "bun": "latest" diff --git a/d2js/js/src/index.js b/d2js/js/src/index.js index 0f8f38e1d..880e48295 100644 --- a/d2js/js/src/index.js +++ b/d2js/js/src/index.js @@ -50,7 +50,7 @@ export class D2 { if (isNode) { this.worker.on("error", (error) => { - console.error("Worker encountered an error:", error.message || error); + console.error("Worker (node) encountered an error:", error.message || error); }); } else { this.worker.onerror = (error) => { diff --git a/d2js/js/src/platform.browser.js b/d2js/js/src/platform.browser.js new file mode 100644 index 000000000..e7f907793 --- /dev/null +++ b/d2js/js/src/platform.browser.js @@ -0,0 +1,29 @@ +import { wasmBinary, wasmExecJs } from "./wasm-loader.browser.js"; + +export async function loadFile(path) { + console.log("loading " + path); + if (path === "./d2.wasm") { + return wasmBinary.buffer; + } + if (path === "./wasm_exec.js") { + return new TextEncoder().encode(wasmExecJs).buffer; + } + throw new Error(`Unexpected file request: ${path}`); +} + +export async function createWorker() { + // Combine wasmExecJs with worker script + const workerResponse = await fetch(new URL("./worker.js", import.meta.url)); + if (!workerResponse.ok) { + throw new Error( + `Failed to load worker.js: ${workerResponse.status} ${workerResponse.statusText}` + ); + } + const workerJs = await workerResponse.text(); + + const blob = new Blob(["(() => {", wasmExecJs, "})();", workerJs], { + type: "application/javascript", + }); + + return new Worker(URL.createObjectURL(blob)); +} diff --git a/d2js/js/src/platform.js b/d2js/js/src/platform.js index 7a767037a..1a607e21d 100644 --- a/d2js/js/src/platform.js +++ b/d2js/js/src/platform.js @@ -1,37 +1 @@ -export async function loadFile(path) { - if (typeof window === "undefined") { - const fs = await import("node:fs/promises"); - const { fileURLToPath } = await import("node:url"); - const { join, dirname } = await import("node:path"); - const __dirname = dirname(fileURLToPath(import.meta.url)); - - try { - return await fs.readFile(join(__dirname, path)); - } catch (err) { - if (err.code === "ENOENT") { - return await fs.readFile(join(__dirname, "../wasm", path.replace("./", ""))); - } - throw err; - } - } - try { - const response = await fetch(new URL(path, import.meta.url)); - return await response.arrayBuffer(); - } catch { - const response = await fetch( - new URL(`../wasm/${path.replace("./", "")}`, import.meta.url) - ); - return await response.arrayBuffer(); - } -} - -export async function createWorker() { - if (typeof window === "undefined") { - const { Worker } = await import("node:worker_threads"); - const { fileURLToPath } = await import("node:url"); - const { join, dirname } = await import("node:path"); - const __dirname = dirname(fileURLToPath(import.meta.url)); - return new Worker(join(__dirname, "worker.js")); - } - return new window.Worker(new URL("./worker.js", import.meta.url)); -} +export * from "./platform.node.js"; diff --git a/d2js/js/src/platform.node.js b/d2js/js/src/platform.node.js new file mode 100644 index 000000000..ffaa65c20 --- /dev/null +++ b/d2js/js/src/platform.node.js @@ -0,0 +1,40 @@ +let nodeModules = null; + +async function loadNodeModules() { + if (!nodeModules) { + nodeModules = { + fs: await import("fs/promises"), + path: await import("path"), + url: await import("url"), + worker: await import("worker_threads"), + }; + } + return nodeModules; +} + +export async function loadFile(path) { + const modules = await loadNodeModules(); + const readFile = modules.fs.readFile; + const { join, dirname } = modules.path; + const { fileURLToPath } = modules.url; + const __dirname = dirname(fileURLToPath(import.meta.url)); + + try { + return await readFile(join(__dirname, path)); + } catch (err) { + if (err.code === "ENOENT") { + return await readFile(join(__dirname, "../../../wasm", path.replace("./", ""))); + } + throw err; + } +} + +export async function createWorker() { + const modules = await loadNodeModules(); + const { Worker } = modules.worker; + const { join, dirname } = modules.path; + const { fileURLToPath } = modules.url; + const __dirname = dirname(fileURLToPath(import.meta.url)); + const workerPath = join(__dirname, "worker.js"); + return new Worker(workerPath); +} diff --git a/d2js/js/src/wasm-loader.node.js b/d2js/js/src/wasm-loader.node.js new file mode 100644 index 000000000..34292bfec --- /dev/null +++ b/d2js/js/src/wasm-loader.node.js @@ -0,0 +1,8 @@ +import { readFile } from "fs/promises"; +import { fileURLToPath } from "url"; +import { dirname, resolve } from "path"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +export async function getWasmBinary() { + return readFile(resolve(__dirname, "./d2.wasm")); +} diff --git a/d2js/js/src/worker.js b/d2js/js/src/worker.js index 08bf6f633..0878809d1 100644 --- a/d2js/js/src/worker.js +++ b/d2js/js/src/worker.js @@ -27,8 +27,6 @@ async function handleMessage(e) { try { if (isNode) { eval(data.wasmExecContent); - } else { - importScripts(data.wasmExecUrl); } d2 = await initWasm(data.wasm); currentPort.postMessage({ type: "ready" }); diff --git a/d2js/js/test/integration/cjs.test.js b/d2js/js/test/integration/cjs.test.js index 527f43d73..bb83d0f0f 100644 --- a/d2js/js/test/integration/cjs.test.js +++ b/d2js/js/test/integration/cjs.test.js @@ -2,7 +2,7 @@ import { expect, test, describe } from "bun:test"; describe("D2 CJS Integration", () => { test("can require and use CJS build", async () => { - const { D2 } = require("../../dist/cjs/index.js"); + const { D2 } = require("../../dist/node-cjs/index.js"); const d2 = new D2(); const result = await d2.compile("x -> y"); expect(result.diagram).toBeDefined(); diff --git a/d2js/js/test/integration/esm.test.js b/d2js/js/test/integration/esm.test.js index 830a62a03..bd1987a24 100644 --- a/d2js/js/test/integration/esm.test.js +++ b/d2js/js/test/integration/esm.test.js @@ -1,5 +1,5 @@ import { expect, test, describe } from "bun:test"; -import { D2 } from "../../dist/esm/index.js"; +import { D2 } from "../../dist/node-esm/index.js"; describe("D2 ESM Integration", () => { test("can import and use ESM build", async () => { diff --git a/d2js/js/test/unit/basic.test.js b/d2js/js/test/unit/basic.test.js index 4b169ec1d..477cbacf4 100644 --- a/d2js/js/test/unit/basic.test.js +++ b/d2js/js/test/unit/basic.test.js @@ -1,5 +1,5 @@ import { expect, test, describe } from "bun:test"; -import { D2 } from "../../src/index.js"; +import { D2 } from "../../dist/node-esm/index.js"; describe("D2 Unit Tests", () => { test("basic compilation works", async () => { From b93dd31ba661c6aab7546bfeec2e5a8c9179de2e Mon Sep 17 00:00:00 2001 From: Alexander Wang Date: Sun, 12 Jan 2025 21:43:03 -0700 Subject: [PATCH 2/3] cleanup stage --- Makefile | 4 ++-- d2js/js/Makefile | 8 ++++++-- d2js/js/build.js | 7 ++++--- make.sh | 8 ++++---- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 0a27720c2..37cf33c18 100644 --- a/Makefile +++ b/Makefile @@ -22,5 +22,5 @@ test: fmt race: fmt prefix "$@" ./ci/test.sh --race ./... .PHONY: js -js: - cd d2js/js && prefix "$@" ./make.sh +js: gen + cd d2js/js && prefix "$@" ./make.sh all diff --git a/d2js/js/Makefile b/d2js/js/Makefile index a8b662792..cc098c626 100644 --- a/d2js/js/Makefile +++ b/d2js/js/Makefile @@ -1,6 +1,6 @@ .POSIX: .PHONY: all -all: fmt build test +all: fmt build test cleanup .PHONY: fmt fmt: node_modules @@ -8,7 +8,7 @@ fmt: node_modules prefix "$@" rm -f yarn.lock .PHONY: build -build: node_modules +build: fmt prefix "$@" ./ci/build.sh .PHONY: test @@ -18,3 +18,7 @@ test: build .PHONY: node_modules node_modules: prefix "$@" bun install $${CI:+--frozen-lockfile} + +.PHONY: cleanup +cleanup: test + prefix "$@" git checkout -- src/platform.js diff --git a/d2js/js/build.js b/d2js/js/build.js index c7a1fcca5..889aaa050 100644 --- a/d2js/js/build.js +++ b/d2js/js/build.js @@ -1,5 +1,5 @@ import { build } from "bun"; -import { copyFile, mkdir, writeFile, readFile, rm } from "node:fs/promises"; +import { copyFile, mkdir, writeFile, readFile, rm, chmod } from "node:fs/promises"; import { join, resolve } from "node:path"; const __dirname = new URL(".", import.meta.url).pathname; @@ -31,11 +31,12 @@ const commonConfig = { async function buildPlatformFile(platform) { const platformContent = platform === "node" - ? "export * from './platform.node.js';" - : "export * from './platform.browser.js';"; + ? `export * from "./platform.node.js";` + : `export * from "./platform.browser.js";`; const platformPath = join(SRC_DIR, "platform.js"); await writeFile(platformPath, platformContent); + await chmod(platformPath, 0o600); } async function buildAndCopy(buildType) { diff --git a/make.sh b/make.sh index 6177fa6f2..9a3161556 100755 --- a/make.sh +++ b/make.sh @@ -14,8 +14,8 @@ if ! go version | grep -q '1.2[0-9]'; then exit 1 fi -if [ "${CI:-}" ]; then - export FORCE_COLOR=1 - npx playwright@1.31.1 install --with-deps chromium -fi +# if [ "${CI:-}" ]; then +# export FORCE_COLOR=1 +# npx playwright@1.31.1 install --with-deps chromium +# fi _make "$@" From 44497adf31166077797ca4b039a1338b146e67a5 Mon Sep 17 00:00:00 2001 From: Alexander Wang Date: Mon, 13 Jan 2025 11:04:48 -0700 Subject: [PATCH 3/3] cleanup --- d2js/js/build.js | 3 +-- d2js/js/src/platform.browser.js | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/d2js/js/build.js b/d2js/js/build.js index 889aaa050..3c1cfeec5 100644 --- a/d2js/js/build.js +++ b/d2js/js/build.js @@ -1,5 +1,5 @@ import { build } from "bun"; -import { copyFile, mkdir, writeFile, readFile, rm, chmod } from "node:fs/promises"; +import { copyFile, mkdir, writeFile, readFile, rm } from "node:fs/promises"; import { join, resolve } from "node:path"; const __dirname = new URL(".", import.meta.url).pathname; @@ -36,7 +36,6 @@ async function buildPlatformFile(platform) { const platformPath = join(SRC_DIR, "platform.js"); await writeFile(platformPath, platformContent); - await chmod(platformPath, 0o600); } async function buildAndCopy(buildType) { diff --git a/d2js/js/src/platform.browser.js b/d2js/js/src/platform.browser.js index e7f907793..34e45749f 100644 --- a/d2js/js/src/platform.browser.js +++ b/d2js/js/src/platform.browser.js @@ -1,7 +1,6 @@ import { wasmBinary, wasmExecJs } from "./wasm-loader.browser.js"; export async function loadFile(path) { - console.log("loading " + path); if (path === "./d2.wasm") { return wasmBinary.buffer; }