Merge pull request #461 from nhooyr/windows-builder-4d88

ci/release: Add docker image and windows .msi
This commit is contained in:
Anmol Sethi 2022-12-18 10:42:08 -08:00 committed by GitHub
commit 1eeedb7fd8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 710 additions and 431 deletions

18
ci/release/Dockerfile Normal file
View file

@ -0,0 +1,18 @@
# https://hub.docker.com/repository/docker/terrastruct/d2
FROM debian:latest
ARG TARGETARCH
COPY ./d2-*-linux-$TARGETARCH.tar.gz /tmp
RUN mkdir -p /usr/local/lib/d2 \
&& tar -C /usr/local/lib/d2 -xzf /tmp/d2-*-linux-"$TARGETARCH".tar.gz \
&& /usr/local/lib/d2/d2-*/scripts/install.sh \
&& rm -Rf /tmp/d2-*-linux-"$TARGETARCH".tar.gz
WORKDIR /root/src
EXPOSE 8080
ENV PORT 8080
ENV HOST 0.0.0.0
ENV BROWSER false
ENTRYPOINT ["/usr/local/bin/d2"]

View file

@ -2,7 +2,7 @@
## _install.sh
The template for the install script in the root of the repository.
The template for the install script in the root of the d2 repository.
### gen_install.sh
@ -23,12 +23,12 @@ it depends on from ../sub/lib.
> variables as we must compile d2 directly on each release target to include dagre.
> See https://github.com/terrastruct/d2/issues/31
Use `--host-only` to build only the release for the host's OS-ARCH pair.
Use `--host-only` to build only the release for the host's `$OS-$ARCH` pair.
### build_docker.sh
Helper script called by build.sh to build D2 on each linux runner inside Docker.
The Dockerfile is in ./builders/Dockerfile
The Dockerfile is in ./linux/Dockerfile
### _build.sh

540
ci/release/aws/ensure.sh Executable file
View file

@ -0,0 +1,540 @@
#!/bin/sh
set -eu
. "$(dirname "$0")/../../../ci/sub/lib.sh"
cd -- "$(dirname "$0")/../../.."
help() {
cat <<EOF
usage: $0 [--dry-run] [--skip-create] [--skip-init] [--copy-id=id.pub]
[--run=jobregex]
$0 creates and ensures the d2 builders in AWS.
EOF
}
main() {
while flag_parse "$@"; do
case "$FLAG" in
h|help)
help
return 0
;;
x)
flag_noarg && shift "$FLAGSHIFT"
set -x
export TRACE=1
;;
dry-run)
flag_noarg && shift "$FLAGSHIFT"
export DRY_RUN=1
;;
copy-id)
flag_nonemptyarg && shift "$FLAGSHIFT"
ID_PUB_PATH=$FLAGARG
;;
run)
flag_reqarg && shift "$FLAGSHIFT"
JOBFILTER="$FLAGARG"
;;
*)
flag_errusage "unrecognized flag $FLAGRAW"
;;
esac
done
shift "$FLAGSHIFT"
if [ $# -gt 0 ]; then
flag_errusage "no arguments are accepted"
fi
if [ -z "${ID_PUB_PATH-}" ]; then
flag_errusage "--copy-id is required"
fi
JOBNAME=create runjob_filter create_remote_hosts
JOBNAME=init && runjob_filter init_remote_hosts
FGCOLOR=2 header summary
echo "export CI_D2_LINUX_AMD64=$CI_D2_LINUX_AMD64"
echo "export CI_D2_LINUX_ARM64=$CI_D2_LINUX_ARM64"
echo "export CI_D2_MACOS_AMD64=$CI_D2_MACOS_AMD64"
echo "export CI_D2_MACOS_ARM64=$CI_D2_MACOS_ARM64"
echo "export CI_D2_WINDOWS_AMD64=$CI_D2_WINDOWS_AMD64"
}
create_remote_hosts() {
bigheader create_remote_hosts
KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName)
KEY_NAME_WINDOWS=windows
VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId)
JOBNAME=$JOBNAME/security-groups runjob_filter create_security_groups
JOBNAME=$JOBNAME/linux/amd64 runjob_filter create_linux_amd64
JOBNAME=$JOBNAME/linux/arm64 runjob_filter create_linux_arm64
JOBNAME=$JOBNAME/macos/amd64 runjob_filter create_macos_amd64
JOBNAME=$JOBNAME/macos/arm64 runjob_filter create_macos_arm64
JOBNAME=$JOBNAME/windows/amd64 runjob_filter create_windows_amd64
}
create_security_groups() {
header security-group
SG_ID=$(aws ec2 describe-security-groups --group-names ssh 2>/dev/null \
| jq -r .SecurityGroups[0].GroupId)
if [ -z "$SG_ID" ]; then
SG_ID=$(sh_c aws ec2 create-security-group \
--group-name ssh \
--description ssh \
--vpc-id "$VPC_ID" | jq -r .GroupId)
fi
header security-group-ingress
SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names ssh \
| jq -r '.SecurityGroups[0].IpPermissions | length')
if [ "$SG_RULES_COUNT" -eq 0 ]; then
sh_c aws ec2 authorize-security-group-ingress \
--group-id "$SG_ID" \
--protocol tcp \
--port 22 \
--cidr 0.0.0.0/0 >/dev/null
fi
header windows-security-group
SG_ID=$(aws ec2 describe-security-groups --group-names windows 2>/dev/null \
| jq -r .SecurityGroups[0].GroupId)
if [ -z "$SG_ID" ]; then
SG_ID=$(sh_c aws ec2 create-security-group \
--group-name windows \
--description windows \
--vpc-id "$VPC_ID" | jq -r .GroupId)
fi
header windows-security-group-ingress
SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names windows \
| jq -r '.SecurityGroups[0].IpPermissions | length')
if [ "$SG_RULES_COUNT" -ne 2 ]; then
sh_c aws ec2 authorize-security-group-ingress \
--group-id "$SG_ID" \
--protocol tcp \
--port 22 \
--cidr 0.0.0.0/0 >/dev/null
sh_c aws ec2 authorize-security-group-ingress \
--group-id "$SG_ID" \
--protocol tcp \
--port 3389 \
--cidr 0.0.0.0/0 >/dev/null
fi
}
create_linux_amd64() {
header linux-amd64
REMOTE_NAME=ci-d2-linux-amd64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-linux-amd64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-0ecc74eca1d66d8a6 \
--count=1 \
--instance-type=t3.small \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=64,VolumeType=gp3}"' \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-linux-amd64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-linux-amd64}]"' >/dev/null
fi
wait_remote_host_ip
log "CI_D2_LINUX_AMD64=ubuntu@$ip"
export CI_D2_LINUX_AMD64=ubuntu@$ip
}
create_linux_arm64() {
header linux-arm64
REMOTE_NAME=ci-d2-linux-arm64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-linux-arm64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-06e2dea2cdda3acda \
--count=1 \
--instance-type=t4g.small \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=64,VolumeType=gp3}"' \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-linux-arm64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-linux-arm64}]"' >/dev/null
fi
wait_remote_host_ip
log "CI_D2_LINUX_ARM64=ubuntu@$ip"
export CI_D2_LINUX_ARM64=ubuntu@$ip
}
create_macos_amd64() {
header macos-amd64-host
MACOS_AMD64_ID=$(aws ec2 describe-hosts --filter 'Name=state,Values=pending,available' 'Name=tag:Name,Values=ci-d2-macos-amd64' | jq -r '.Hosts[].HostId')
if [ -z "$MACOS_AMD64_ID" ]; then
MACOS_AMD64_ID=$(sh_c aws ec2 allocate-hosts --instance-type mac1.metal --quantity 1 --availability-zone us-west-2a \
--tag-specifications '"ResourceType=dedicated-host,Tags=[{Key=Name,Value=ci-d2-macos-amd64}]"' \
| jq -r .HostIds[0])
fi
header macos-amd64
REMOTE_NAME=ci-d2-macos-amd64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-macos-amd64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-0dd2ded7568750663 \
--count=1 \
--instance-type=mac1.metal \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
--placement "Tenancy=host,HostId=$MACOS_AMD64_ID" \
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=100,VolumeType=gp3}"' \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-macos-amd64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-macos-amd64}]"' >/dev/null
fi
wait_remote_host_ip
log "CI_D2_MACOS_AMD64=ec2-user@$ip"
export CI_D2_MACOS_AMD64=ec2-user@$ip
}
create_macos_arm64() {
header macos-arm64-host
MACOS_ARM64_ID=$(aws ec2 describe-hosts --filter 'Name=state,Values=pending,available' 'Name=tag:Name,Values=ci-d2-macos-arm64' | jq -r '.Hosts[].HostId')
if [ -z "$MACOS_ARM64_ID" ]; then
MACOS_ARM64_ID=$(sh_c aws ec2 allocate-hosts --instance-type mac2.metal --quantity 1 --availability-zone us-west-2a \
--tag-specifications '"ResourceType=dedicated-host,Tags=[{Key=Name,Value=ci-d2-macos-arm64}]"' \
| jq -r .HostIds[0])
fi
header macos-arm64
REMOTE_NAME=ci-d2-macos-arm64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-macos-arm64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-0af0516ff2c43dbbe \
--count=1 \
--instance-type=mac2.metal \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
--placement "Tenancy=host,HostId=$MACOS_ARM64_ID" \
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=100,VolumeType=gp3}"' \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-macos-arm64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-macos-arm64}]"' >/dev/null
fi
wait_remote_host_ip
log "CI_D2_MACOS_ARM64=ec2-user@$ip"
export CI_D2_MACOS_ARM64=ec2-user@$ip
}
create_windows_amd64() {
header windows-amd64
REMOTE_NAME=ci-d2-windows-amd64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-0c5300e833c2b32f3 \
--count=1 \
--instance-type=t3.medium \
--security-groups=windows \
"--key-name=$KEY_NAME_WINDOWS" \
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=64,VolumeType=gp3}"' \
--tag-specifications "'ResourceType=instance,Tags=[{Key=Name,Value=$REMOTE_NAME}]'" \
"'ResourceType=volume,Tags=[{Key=Name,Value=$REMOTE_NAME}]'" >/dev/null
fi
wait_remote_host_ip
log "CI_D2_WINDOWS_AMD64=Administrator@$ip"
export CI_D2_WINDOWS_AMD64=Administrator@$ip
}
wait_remote_host_ip() {
while true; do
ip=$(sh_c aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
| jq -r '.Reservations[].Instances[].PublicIpAddress')
if [ -n "$ip" ]; then
alloc_static_ip
ip=$(sh_c aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
| jq -r '.Reservations[].Instances[].PublicIpAddress')
break
fi
sleep 5
done
}
alloc_static_ip() {
allocation_id=$(aws ec2 describe-addresses --filters "Name=tag:Name,Values=$REMOTE_NAME" | jq -r '.Addresses[].AllocationId')
if [ -z "$allocation_id" ]; then
sh_c aws ec2 allocate-address --tag-specifications "'ResourceType=elastic-ip,Tags=[{Key=Name,Value=$REMOTE_NAME}]'"
allocation_id=$(aws ec2 describe-addresses --filters "Name=tag:Name,Values=$REMOTE_NAME" | jq -r '.Addresses[].AllocationId')
fi
instance_id=$(aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
| jq -r '.Reservations[].Instances[].InstanceId')
aws ec2 associate-address --instance-id "$instance_id" --allocation-id "$allocation_id"
}
init_remote_hosts() {
bigheader init_remote_hosts
JOBNAME=$JOBNAME/linux/amd64 runjob_filter REMOTE_HOST=$CI_D2_LINUX_AMD64 REMOTE_NAME=ci-d2-linux-amd64 init_remote_linux
JOBNAME=$JOBNAME/linux/arm64 runjob_filter REMOTE_HOST=$CI_D2_LINUX_ARM64 REMOTE_NAME=ci-d2-linux-arm64 init_remote_linux
JOBNAME=$JOBNAME/macos/amd64 runjob_filter REMOTE_HOST=$CI_D2_MACOS_AMD64 REMOTE_NAME=ci-d2-macos-amd64 init_remote_macos
JOBNAME=$JOBNAME/macos/arm64 runjob_filter REMOTE_HOST=$CI_D2_MACOS_ARM64 REMOTE_NAME=ci-d2-macos-arm64 init_remote_macos
JOBNAME=$JOBNAME/windows/amd64 runjob_filter REMOTE_HOST=$CI_D2_WINDOWS_AMD64 REMOTE_NAME=ci-d2-windows-amd64 init_remote_windows
# Windows and AWS SSM both defeated me.
FGCOLOR=3 bigheader "WARNING: WINDOWS INITIALIZATION MUST BE COMPLETED MANUALLY OVER RDP AND POWERSHELL!"
}
init_remote_linux() {
header "$REMOTE_NAME"
wait_remote_host
sh_c ssh_copy_id -i="$ID_PUB_PATH" "$REMOTE_HOST"
sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
set -eux
export DEBIAN_FRONTEND=noninteractive
sudo -E apt-get update -y
sudo -E apt-get dist-upgrade -y
sudo -E apt-get update -y
sudo -E apt-get install -y build-essential rsync
# Docker from https://docs.docker.com/engine/install/ubuntu/
sudo -E apt-get -y install \
ca-certificates \
curl \
gnupg \
lsb-release
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --yes --dearmor -o /etc/apt/keyrings/docker.gpg
echo \
"deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
\$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo -E apt-get update -y
sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
sudo groupadd docker || true
sudo usermod -aG docker \$USER
mkdir -p \$HOME/.local/bin
mkdir -p \$HOME/.local/share/man
EOF
init_remote_env
sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
set -eux
export DEBIAN_FRONTEND=noninteractive
sudo -E apt-get autoremove -y
EOF
sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
}
init_remote_macos() {
header "$REMOTE_NAME"
wait_remote_host
sh_c ssh "$REMOTE_HOST" '"/bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\""'
if sh_c ssh "$REMOTE_HOST" uname -m | grep -qF arm64; then
shellenv=$(sh_c ssh "$REMOTE_HOST" /opt/homebrew/bin/brew shellenv)
else
shellenv=$(sh_c ssh "$REMOTE_HOST" /usr/local/bin/brew shellenv)
fi
if ! echo "$shellenv" | sh_c ssh "$REMOTE_HOST" "IFS= read -r regex\; \"grep -qF \\\"\\\$regex\\\" ~/.zshrc\""; then
echo "$shellenv" | sh_c ssh "$REMOTE_HOST" "\"(echo && cat) >> ~/.zshrc\""
fi
if ! sh_c ssh "$REMOTE_HOST" "'grep -qF \\\$HOME/.local ~/.zshrc'"; then
sh_c ssh "$REMOTE_HOST" "\"(echo && cat) >> ~/.zshrc\"" <<EOF
PATH=\$HOME/.local/bin:\$PATH
MANPATH=\$HOME/.local/share/man:\$MANPATH
EOF
fi
init_remote_env
sh_c ssh "$REMOTE_HOST" brew update
sh_c ssh "$REMOTE_HOST" brew upgrade
sh_c ssh "$REMOTE_HOST" brew install go rsync
sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
}
init_remote_env() {
sh_c ssh "$REMOTE_HOST" '"rm -f ~/.ssh/environment"'
sh_c ssh "$REMOTE_HOST" '"echo PATH=\$(echo \"echo \\\$PATH\" | \"\$SHELL\" -ils) >\$HOME/.ssh/environment"'
sh_c ssh "$REMOTE_HOST" '"echo MANPATH=\$(echo \"echo \\\$MANPATH\" | \"\$SHELL\" -ils) >>\$HOME/.ssh/environment"'
sh_c ssh "$REMOTE_HOST" "sudo sed -i.bak '\"s/#PermitUserEnvironment no/PermitUserEnvironment yes/\"' /etc/ssh/sshd_config"
if sh_c ssh "$REMOTE_HOST" uname | grep -qF Darwin; then
sh_c ssh "$REMOTE_HOST" "sudo launchctl stop com.openssh.sshd"
else
sh_c ssh "$REMOTE_HOST" "sudo systemctl restart sshd"
fi
}
wait_remote_host() {
ssh-keygen -R "${REMOTE_HOST##*@}"
while true; do
if sh_c ssh "$REMOTE_HOST" true; then
break
fi
sleep 5
done
}
wait_remote_host_windows() {
instance_id=$(aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
| jq -r '.Reservations[].Instances[].InstanceId')
while true; do
if sh_c aws ssm start-session --target "$instance_id" \
--document-name 'AWS-StartNonInteractiveCommand' \
--parameters "'{\"command\": [\"echo true\"]}'"; then
break
fi
sleep 5
done
}
init_remote_windows() {
header "$REMOTE_NAME"
wait_remote_host_windows
init_ps1=$(cat <<EOF
\$ProgressPreference = 'SilentlyContinue'
# Bootstrap PowerShell v7
if ((\$PSVersionTable.PSVersion).Major -eq 5) {
Invoke-WebRequest -Uri https://www.nuget.org/api/v2/package/Microsoft.UI.Xaml/2.7.3 -OutFile .\microsoft.ui.xaml.2.7.3.zip
Expand-Archive -Force .\microsoft.ui.xaml.2.7.3.zip
Add-AppxPackage .\microsoft.ui.xaml.2.7.3\tools\AppX\x64\Release\Microsoft.UI.Xaml.2.7.appx
Invoke-WebRequest -Uri https://github.com/microsoft/winget-cli/releases/download/v1.3.2691/Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle -OutFile .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle
Invoke-WebRequest -Uri https://github.com/microsoft/winget-cli/releases/download/v1.3.2691/7bcb1a0ab33340daa57fa5b81faec616_License1.xml -OutFile .\7bcb1a0ab33340daa57fa5b81faec616_License1.xml
Invoke-WebRequest -Uri https://aka.ms/Microsoft.VCLibs.x64.14.00.Desktop.appx -OutFile Microsoft.VCLibs.x64.14.00.Desktop.appx
Add-AppxProvisionedPackage -online -PackagePath .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle -LicensePath .\7bcb1a0ab33340daa57fa5b81faec616_License1.xml -DependencyPackagePath Microsoft.VCLibs.x64.14.00.Desktop.appx
Add-AppxPackage .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle
winget install --silent --accept-package-agreements --accept-source-agreements Microsoft.DotNet.SDK.7
# Refresh env.
\$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
dotnet tool install --global PowerShell --version 7.3.1
pwsh -c 'Enable-ExperimentalFeature PSNativeCommandErrorActionPreference'
pwsh .\Desktop\init.ps1
Exit
}
Set-StrictMode -Version Latest
\$ErrorActionPreference = "Stop"
\$PSNativeCommandUseErrorActionPreference = \$true
if (-Not (Get-Command wix -errorAction SilentlyContinue)) {
dotnet tool install --global wix --version 4.0.0-preview.1
}
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
Start-Service sshd
Set-Service -Name sshd -StartupType 'Automatic'
New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" -Name DefaultShell -Value "C:\Users\Administrator\.dotnet\tools\pwsh.exe" -PropertyType String -Force
ConvertFrom-Json -InputObject @'
$(perl -pe 's#\n#\r\n#' "$ID_PUB_PATH" | jq -Rs .)
'@ | Out-File -Encoding utf8 "\$env:ProgramData\ssh\administrators_authorized_keys"
# utf8BOM -> utf8: https://stackoverflow.com/a/34969243/4283659
\$null = New-Item -Force "\$env:ProgramData\ssh\administrators_authorized_keys" -Value (Get-Content -Path "\$env:ProgramData\ssh\administrators_authorized_keys" | Out-String)
get-acl "\$env:ProgramData\ssh\ssh_host_rsa_key" | set-acl "\$env:ProgramData\ssh\administrators_authorized_keys"
if (-Not (Test-Path -Path C:\msys64)) {
Invoke-WebRequest -Uri "https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-x86_64-20221028.exe" -OutFile "./msys2-x86_64.exe"
./msys2-x86_64.exe install --default-answer --confirm-command --root C:\msys64
}
C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c 'pacman -Sy --noconfirm base-devel vim rsync'
C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c 'curl -fsSL https://d2lang.com/install.sh | sh -s -- --tala'
C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c 'd2 --version'
\$path = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name Path).Path
if (\$path -notlike '*C:\msys64\usr\bin*') {
\$path = "\$path;C:\msys64\usr\bin"
Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name Path -Value \$path
}
if (\$path -notlike '*C:\msys64\usr\local\bin*') {
\$path = "\$path;C:\msys64\usr\local\bin"
Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name Path -Value \$path
}
(Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name Path).Path
Restart-Computer
EOF
# To run a POSIX script:
# ssh "$CI_D2_WINDOWS_AMD64" sh -s -- <<EOF
# wix --version
# EOF
# To run a command in a pure MSYS2 shell:
# ssh "$CI_D2_WINDOWS_AMD64" 'C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c "\"d2 --version\""'
# To run a pure MSYS2 shell:
# ssh -t "$CI_D2_WINDOWS_AMD64" 'C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64'
# In case MSYS2 improves in the future and allows for noninteractive commands the
# following will set the OpenSSH shell to MSYS2 instead of PowerShell.
#
# Right now, setting MSYS2 to the DefaultShell like this will make it start bash in
# interactive mode always. Even for ssh "$CI_D2_WINDOWS_AMD64" echo hi. And so you'll end
# up with a blank prompt on which to input commands instead of having it execute the
# command you passed in via ssh.
#
# PowerShell as the default is better anyway as it gives us access to both the UNIX
# userspace and Windows tools like wix/dotnet/winget.
#
# To set:
# <<EOF
# echo '@C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64' | Out-File C:\msys64\sshd_default_shell.cmd
# # utf8BOM -> utf8: https://stackoverflow.com/a/34969243/4283659
# \$null = New-Item -Force C:\msys64\sshd_default_shell.cmd -Value (Get-Content -Path C:\msys64\sshd_default_shell.cmd | Out-String)
# Set-ItemProperty -Path HKLM:\SOFTWARE\OpenSSH -Name DefaultShell -Value C:\msys64\sshd_default_shell.cmd
# EOF
#
# To undo:
# <<EOF
# Remove-ItemProperty -Path HKLM:\SOFTWARE\OpenSSH -Name DefaultShell
# rm C:\msys64\sshd_default_shell.cmd
# EOF
)
gen_init_ps1=$(cat <<EOF
ConvertFrom-Json -InputObject @'
$(printf %s "$init_ps1" | perl -pe 'chomp if eof' | perl -pe 's#\n#\r\n#' | jq -Rs .)
'@ | Out-File -Encoding utf8 C:\Users\Administrator\Desktop\init.ps1; C:\Users\Administrator\Desktop\init.ps1
EOF
)
# Windows and AWS SSM both defeated me.
FGCOLOR=3 bigheader "WARNING: WINDOWS INITIALIZATION MUST BE COMPLETED MANUALLY OVER RDP AND POWERSHELL!"
warn '1. Obtain Windows RDP password with:'
echo " aws ec2 get-password-data --instance-id \$(aws ec2 describe-instances --filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" | jq -r '.Reservations[].Instances[].InstanceId') --priv-launch-key windows.pem | jq -r .PasswordData" >&2
warn "2. RDP into $REMOTE_HOST and open PowerShell."
warn '3. Generate and execute C:\Users\Administrator\Desktop\init.ps1 with:'
printf '%s\n' "$gen_init_ps1" >&2
warn '4. Run the following to be notified once installation is successful:'
cat <<EOF
ssh-keygen -R ${REMOTE_HOST##*@} && until ssh $REMOTE_HOST d2 --version; do echo 'failed: retrying in 5s' && sleep 5; done && printf 'success\a\n'
EOF
}
main "$@"

View file

@ -1,7 +1,7 @@
#!/bin/sh
set -eu
. "$(dirname "$0")/../../../ci/sub/lib.sh"
cd -- "$(dirname "$0")/../../.."
. ./ci/sub/lib.sh
help() {
cat <<EOF
@ -33,10 +33,11 @@ main() {
done
shift "$FLAGSHIFT"
REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER; runjob linux-amd64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER; runjob linux-arm64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER; runjob macos-amd64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER; runjob macos-arm64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$CI_HOST_D2_LINUX_AMD64 && runjob linux-amd64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$CI_HOST_D2_LINUX_ARM64 && runjob linux-arm64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$CI_HOST_D2_MACOS_AMD64 && runjob macos-amd64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$CI_HOST_D2_MACOS_ARM64 && runjob macos-arm64 ssh "$REMOTE_HOST" "$@"
REMOTE_HOST=$CI_HOST_D2_WINDOWS_AMD64 && runjob macos-arm64 ssh "$REMOTE_HOST" "$@"
}
main "$@"

View file

@ -20,11 +20,12 @@ Flags:
changed something and need to force rebuild, use this flag.
--local
By default build.sh uses \$TSTRUCT_MACOS_AMD64_BUILDER, \$TSTRUCT_MACOS_ARM64_BUILDER,
\$TSTRUCT_LINUX_AMD64_BUILDER and \$TSTRUCT_LINUX_ARM64_BUILDER to build the release
By default build.sh uses \$CI_D2_LINUX_AMD64, \$CI_D2_LINUX_ARM64,
\$CI_D2_MACOS_AMD64 and \$CI_D2_MACOS_ARM64 to build the release
archives. It's required for now due to the following issue:
https://github.com/terrastruct/d2/issues/31 With --local, build.sh will cross compile
locally. warning: This is only for testing purposes, do not use in production!
https://github.com/terrastruct/d2/issues/31
With --local, build.sh will cross compile locally. warning: This is only for testing
purposes, do not use in production!
--host-only
Use to build the release archive for the host OS-ARCH only. All logging is done to stderr
@ -45,6 +46,13 @@ Flags:
--uninstall
Ensure a release using --host-only and uninstall it.
--push-docker
Push the built docker image. Unfortunately dockerx requires the multi-arch images be
pushed if required in the same invocation as build. dockerx cannot load multi-arch
images into the daemon for push later. It's not slow though to use --push-docker after
building the image as nearly all artifacts are cached.
Automatically set if called from release.sh
EOF
}
@ -69,7 +77,7 @@ main() {
;;
run)
flag_reqarg && shift "$FLAGSHIFT"
JOBFILTER="$FLAGARG"
JOBFILTER=$FLAGARG
;;
host-only)
flag_noarg && shift "$FLAGSHIFT"
@ -96,6 +104,10 @@ main() {
HOST_ONLY=1
LOCAL=1
;;
push-docker)
flag_noarg && shift "$FLAGSHIFT"
PUSH_DOCKER=1
;;
*)
flag_errusage "unrecognized flag $FLAGRAW"
;;
@ -108,10 +120,13 @@ main() {
VERSION=${VERSION:-$(git_describe_ref)}
BUILD_DIR=ci/release/build/$VERSION
sh_c mkdir -p "$BUILD_DIR"
sh_c rm -f ci/release/build/latest
sh_c ln -s "$VERSION" ci/release/build/latest
if [ -n "${HOST_ONLY-}" ]; then
ensure_os
ensure_arch
runjob "$OS-$ARCH" "build"
runjob "$OS/$ARCH" "build"
if [ -n "${INSTALL-}" ]; then
sh_c make -sC "ci/release/build/$VERSION/$OS-$ARCH/d2-$VERSION" install
@ -121,12 +136,16 @@ main() {
return 0
fi
runjob linux-amd64 'OS=linux ARCH=amd64 build' &
runjob linux-arm64 'OS=linux ARCH=arm64 build' &
runjob macos-amd64 'OS=macos ARCH=amd64 build' &
runjob macos-arm64 'OS=macos ARCH=arm64 build' &
runjob windows-amd64 'OS=windows ARCH=amd64 build' &
runjob windows-arm64 'OS=windows ARCH=arm64 build' &
runjob linux/amd64 'OS=linux ARCH=amd64 build' &
runjob linux/arm64 'OS=linux ARCH=arm64 build' &
runjob macos/amd64 'OS=macos ARCH=amd64 build' &
runjob macos/arm64 'OS=macos ARCH=arm64 build' &
runjob windows/amd64 'OS=windows ARCH=amd64 build' &
runjob windows/arm64 'OS=windows ARCH=arm64 build' &
waitjobs
runjob linux/dockerimage 'OS=linux build_docker_image' &
runjob windows/amd64/msi 'OS=windows ARCH=amd64 build_windows_msi' &
waitjobs
}
@ -148,10 +167,10 @@ build() {
macos)
case $ARCH in
amd64)
REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER build_remote_macos
REMOTE_HOST=$CI_D2_MACOS_AMD64 build_remote_macos
;;
arm64)
REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER build_remote_macos
REMOTE_HOST=$CI_D2_MACOS_ARM64 build_remote_macos
;;
*)
warn "no builder for OS=$OS ARCH=$ARCH, building locally..."
@ -162,10 +181,10 @@ build() {
linux)
case $ARCH in
amd64)
REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER build_remote_linux
REMOTE_HOST=$CI_D2_LINUX_AMD64 build_remote_linux
;;
arm64)
REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER build_remote_linux
REMOTE_HOST=$CI_D2_LINUX_ARM64 build_remote_linux
;;
*)
warn "no builder for OS=$OS ARCH=$ARCH, building locally..."
@ -218,13 +237,39 @@ VERSION=$VERSION \
OS=$OS \
ARCH=$ARCH \
ARCHIVE=$ARCHIVE \
./src/d2/ci/release/build_docker.sh"
./src/d2/ci/release/build_in_docker.sh"
sh_c mkdir -p "$HW_BUILD_DIR"
sh_c rsync --archive --human-readable "$REMOTE_HOST:src/d2/$ARCHIVE" "$ARCHIVE"
)}
ssh() {
command ssh -o='StrictHostKeyChecking=accept-new' "$@"
build_docker_image() {
D2_DOCKER_IMAGE=${D2_DOCKER_IMAGE:-terrastruct/d2}
flags='--load'
if [ -n "${PUSH_DOCKER-}" -o -n "${RELEASE-}" ]; then
flags='--push --platform linux/amd64,linux/arm64'
fi
sh_c docker buildx build $flags -t "$D2_DOCKER_IMAGE:$VERSION" -t "$D2_DOCKER_IMAGE:latest" --build-arg "VERSION=$VERSION" -f ./ci/release/Dockerfile "./ci/release/build/$VERSION"
}
build_windows_msi() {
REMOTE_HOST=$CI_D2_WINDOWS_AMD64
ln -sf "../build/$VERSION/windows-amd64/d2-$VERSION/bin/d2.exe" ./ci/release/windows/d2.exe
sh_c rsync --archive --human-readable --copy-links --delete ./ci/release/windows/ "'$REMOTE_HOST:windows\'"
if ! echo "$VERSION" | grep '[0-9]\.[0-9].[0-9]'; then
WIX_VERSION=0.0.0
else
WIX_VERSION=$VERSION
fi
sh_c ssh "$REMOTE_HOST" "'cd .\\windows && wix build -arch x64 -d D2Version=$WIX_VERSION .\d2.wxs'"
# --files-from shouldn't be necessary but for some reason selecting d2.msi directly
# makes rsync error with:
# ERROR: rejecting unrequested file-list name: .\\windows\\d2.msi
# rsync error: requested action not supported (code 4) at flist.c(1027) [Receiver=3.2.7]
rsync_files=$(mktempd)/rsync-files
echo d2.msi >$rsync_files
sh_c rsync --archive --human-readable --files-from "$rsync_files" "'$REMOTE_HOST:windows\\'" "./ci/release/build/$VERSION/d2-$VERSION-$OS-$ARCH.msi"
}
main "$@"

View file

@ -0,0 +1 @@
**/d2*/

View file

@ -5,7 +5,7 @@ cd -- "$(dirname "$0")/../.."
tag="$(sh_c docker build \
--build-arg GOVERSION="1.19.3.linux-$ARCH" \
-qf ./ci/release/builders/Dockerfile ./ci/release/builders)"
-qf ./ci/release/linux/Dockerfile ./ci/release/linux)"
docker_run \
-e DRY_RUN \
-e HW_BUILD_DIR \

View file

@ -1,16 +0,0 @@
FROM centos:7
ARG GOVERSION=
RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.tar.gz" >/tmp/go.tar.gz
RUN tar -C /usr/local -xzf /tmp/go.tar.gz
ENV PATH="/usr/local/go/bin:$PATH"
RUN yum install -y rsync wget
RUN yum groupinstall -y 'Development Tools'
RUN curl -fsSL https://ftp.gnu.org/gnu/gcc/gcc-5.2.0/gcc-5.2.0.tar.gz >/tmp/gcc.tar.gz
RUN tar -C /usr/local -xzf /tmp/gcc.tar.gz
RUN cd /usr/local/gcc-5.2.0 && ./contrib/download_prerequisites && mkdir -p build \
&& cd build && ../configure --disable-multilib && make && make install

View file

@ -1,57 +0,0 @@
#!/bin/sh
set -eu
cd -- "$(dirname "$0")/../../.."
. ./ci/sub/lib.sh
help() {
cat <<EOF
usage: $0 [--dry-run] -i keys.pub
$0 copies keys.pub to each builder and then deduplicates its .authorized_keys.
EOF
}
main() {
while flag_parse "$@"; do
case "$FLAG" in
h|help)
help
return 0
;;
dry-run)
flag_noarg && shift "$FLAGSHIFT"
DRY_RUN=1
;;
i)
flag_nonemptyarg && shift "$FLAGSHIFT"
KEY_FILE=$FLAGARG
;;
*)
flag_errusage "unrecognized flag $FLAGRAW"
;;
esac
done
shift "$FLAGSHIFT"
if [ -z "${KEY_FILE-}" ]; then
echoerr "-i is required"
exit 1
fi
header linux-amd64
REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER copy_keys
header linux-arm64
REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER copy_keys
header macos-amd64
REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER copy_keys
header macos-arm64
REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER copy_keys
}
copy_keys() {
sh_c ssh-copy-id -fi "$KEY_FILE" "$REMOTE_HOST"
sh_c ssh "$REMOTE_HOST" 'cat .ssh/authorized_keys \| sort -u \> .ssh/authorized_keys.dedup'
sh_c ssh "$REMOTE_HOST" 'cp .ssh/authorized_keys.dedup .ssh/authorized_keys'
sh_c ssh "$REMOTE_HOST" 'rm .ssh/authorized_keys.dedup'
}
main "$@"

View file

@ -1,294 +0,0 @@
#!/bin/sh
set -eu
cd -- "$(dirname "$0")/../../.."
. ./ci/sub/lib.sh
help() {
cat <<EOF
usage: $0 [--dry-run] [--skip-create]
$0 creates and ensures the d2 builders in AWS.
EOF
}
main() {
while flag_parse "$@"; do
case "$FLAG" in
h|help)
help
return 0
;;
dry-run)
flag_noarg && shift "$FLAGSHIFT"
DRY_RUN=1
;;
skip-create)
flag_noarg && shift "$FLAGSHIFT"
SKIP_CREATE=1
;;
*)
flag_errusage "unrecognized flag $FLAGRAW"
;;
esac
done
shift "$FLAGSHIFT"
if [ $# -gt 0 ]; then
flag_errusage "no arguments are accepted"
fi
if [ -z "${SKIP_CREATE-}" ]; then
create_remote_hosts
fi
init_remote_hosts
}
create_remote_hosts() {
KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName)
VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId)
header security-group
SG_ID=$(aws ec2 describe-security-groups --group-names ssh 2>/dev/null \
| jq -r .SecurityGroups[0].GroupId)
if [ -z "$SG_ID" ]; then
SG_ID=$(sh_c aws ec2 create-security-group \
--group-name ssh \
--description ssh \
--vpc-id "$VPC_ID" | jq -r .GroupId)
fi
header security-group-ingress
SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names ssh \
| jq -r '.SecurityGroups[0].IpPermissions | length')
if [ "$SG_RULES_COUNT" -eq 0 ]; then
sh_c aws ec2 authorize-security-group-ingress \
--group-id "$SG_ID" \
--protocol tcp \
--port 22 \
--cidr 0.0.0.0/0 >/dev/null
fi
header linux-amd64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-amd64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-071e6cafc48327ca2 \
--count=1 \
--instance-type=t2.small \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' >/dev/null
fi
while true; do
dnsname=$(sh_c aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-amd64' \
| jq -r '.Reservations[].Instances[].PublicDnsName')
if [ -n "$dnsname" ]; then
log "TSTRUCT_LINUX_AMD64_BUILDER=admin@$dnsname"
export TSTRUCT_LINUX_AMD64_BUILDER=admin@$dnsname
break
fi
sleep 5
done
header linux-arm64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-arm64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-0e67506f183e5ab60 \
--count=1 \
--instance-type=t4g.small \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' >/dev/null
fi
while true; do
dnsname=$(sh_c aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-arm64' \
| jq -r '.Reservations[].Instances[].PublicDnsName')
if [ -n "$dnsname" ]; then
log "TSTRUCT_LINUX_ARM64_BUILDER=admin@$dnsname"
export TSTRUCT_LINUX_ARM64_BUILDER=admin@$dnsname
break
fi
sleep 5
done
header "macos-amd64-host"
MACOS_AMD64_HOST_ID=$(aws ec2 describe-hosts --filter 'Name=state,Values=pending,available' 'Name=tag:Name,Values=d2-builder-macos-amd64' | jq -r '.Hosts[].HostId')
if [ -z "$MACOS_AMD64_HOST_ID" ]; then
MACOS_AMD64_HOST_ID=$(sh_c aws ec2 allocate-hosts --instance-type mac1.metal --quantity 1 --availability-zone us-west-2a \
--tag-specifications '"ResourceType=dedicated-host,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' \
| jq -r .HostIds[0])
fi
header "macos-arm64-host"
MACOS_ARM64_HOST_ID=$(aws ec2 describe-hosts --filter 'Name=state,Values=pending,available' 'Name=tag:Name,Values=d2-builder-macos-arm64' | jq -r '.Hosts[].HostId')
if [ -z "$MACOS_ARM64_HOST_ID" ]; then
MACOS_ARM64_HOST_ID=$(sh_c aws ec2 allocate-hosts --instance-type mac2.metal --quantity 1 --availability-zone us-west-2a \
--tag-specifications '"ResourceType=dedicated-host,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' \
| jq -r .HostIds[0])
fi
header macos-amd64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-macos-amd64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-0dd2ded7568750663 \
--count=1 \
--instance-type=mac1.metal \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--placement "Tenancy=host,HostId=$MACOS_AMD64_HOST_ID" \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' >/dev/null
fi
while true; do
dnsname=$(sh_c aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-macos-amd64' \
| jq -r '.Reservations[].Instances[].PublicDnsName')
if [ -n "$dnsname" ]; then
log "TSTRUCT_MACOS_AMD64_BUILDER=ec2-user@$dnsname"
export TSTRUCT_MACOS_AMD64_BUILDER=ec2-user@$dnsname
break
fi
sleep 5
done
header macos-arm64
state=$(aws ec2 describe-instances --filters \
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-macos-arm64' \
| jq -r '.Reservations[].Instances[].State.Name')
if [ -z "$state" ]; then
sh_c aws ec2 run-instances \
--image-id=ami-0af0516ff2c43dbbe \
--count=1 \
--instance-type=mac2.metal \
--security-groups=ssh \
"--key-name=$KEY_NAME" \
--placement "Tenancy=host,HostId=$MACOS_ARM64_HOST_ID" \
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' \
'"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' >/dev/null
fi
while true; do
dnsname=$(sh_c aws ec2 describe-instances \
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-macos-arm64' \
| jq -r '.Reservations[].Instances[].PublicDnsName')
if [ -n "$dnsname" ]; then
log "TSTRUCT_MACOS_ARM64_BUILDER=ec2-user@$dnsname"
export TSTRUCT_MACOS_ARM64_BUILDER=ec2-user@$dnsname
break
fi
sleep 5
done
}
init_remote_hosts() {
header linux-amd64
REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER init_remote_linux
header linux-arm64
REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER init_remote_linux
header macos-amd64
REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER init_remote_macos
header macos-arm64
REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER init_remote_macos
FGCOLOR=2 header summary
log "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER"
log "export TSTRUCT_LINUX_ARM64_BUILDER=$TSTRUCT_LINUX_ARM64_BUILDER"
log "export TSTRUCT_MACOS_AMD64_BUILDER=$TSTRUCT_MACOS_AMD64_BUILDER"
log "export TSTRUCT_MACOS_ARM64_BUILDER=$TSTRUCT_MACOS_ARM64_BUILDER"
}
init_remote_linux() {
wait_remote_host
sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
set -eux
export DEBIAN_FRONTEND=noninteractive
sudo -E apt-get update -y
sudo -E apt-get dist-upgrade -y
sudo -E apt-get install -y build-essential rsync
# Docker from https://docs.docker.com/engine/install/debian/
sudo -E apt-get -y install \
ca-certificates \
curl \
gnupg \
lsb-release
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg
echo \
"deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
\$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo -E apt-get update -y
sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
sudo groupadd docker || true
sudo usermod -aG docker \$USER
mkdir -p \$HOME/.local/bin
mkdir -p \$HOME/.local/share/man
EOF
init_remote_env
sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
}
init_remote_macos() {
wait_remote_host
sh_c ssh "$REMOTE_HOST" '"/bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\""'
if sh_c ssh "$REMOTE_HOST" uname -m | grep -qF arm64; then
shellenv=$(sh_c ssh "$REMOTE_HOST" /opt/homebrew/bin/brew shellenv)
else
shellenv=$(sh_c ssh "$REMOTE_HOST" /usr/local/bin/brew shellenv)
fi
if ! echo "$shellenv" | sh_c ssh "$REMOTE_HOST" "IFS= read -r regex\; \"grep -qF \\\"\\\$regex\\\" ~/.zshrc\""; then
echo "$shellenv" | sh_c ssh "$REMOTE_HOST" "\"(echo && cat) >> ~/.zshrc\""
fi
if ! sh_c ssh "$REMOTE_HOST" "'grep -qF \\\$HOME/.local ~/.zshrc'"; then
sh_c ssh "$REMOTE_HOST" "\"(echo && cat) >> ~/.zshrc\"" <<EOF
PATH=\$HOME/.local/bin:\$PATH
MANPATH=\$HOME/.local/share/man:\$MANPATH
EOF
fi
init_remote_env
sh_c ssh "$REMOTE_HOST" brew update
sh_c ssh "$REMOTE_HOST" brew upgrade
sh_c ssh "$REMOTE_HOST" brew install go rsync
}
init_remote_env() {
sh_c ssh "$REMOTE_HOST" '"rm -f ~/.ssh/environment"'
sh_c ssh "$REMOTE_HOST" '"echo PATH=\$(echo \"echo \\\$PATH\" | \"\$SHELL\" -ils) >\$HOME/.ssh/environment"'
sh_c ssh "$REMOTE_HOST" '"echo MANPATH=\$(echo \"echo \\\$MANPATH\" | \"\$SHELL\" -ils) >>\$HOME/.ssh/environment"'
sh_c ssh "$REMOTE_HOST" "sudo sed -i.bak '\"s/#PermitUserEnvironment no/PermitUserEnvironment yes/\"' /etc/ssh/sshd_config"
if sh_c ssh "$REMOTE_HOST" uname | grep -qF Darwin; then
sh_c ssh "$REMOTE_HOST" "sudo launchctl stop com.openssh.sshd"
else
sh_c ssh "$REMOTE_HOST" "sudo systemctl restart sshd"
fi
}
wait_remote_host() {
while true; do
if sh_c ssh "$REMOTE_HOST" true; then
break
fi
sleep 5
done
}
main "$@"

View file

@ -1,20 +1,18 @@
#### Features 🚀
- Diagram padding can now can be configured in the CLI (default 100px).
[https://github.com/terrastruct/d2/pull/431](https://github.com/terrastruct/d2/pull/431)
- Diagram padding can now can be configured in the CLI (default 100px). [https://github.com/terrastruct/d2/pull/431](https://github.com/terrastruct/d2/pull/431)
- Connection label backgrounds can now be set with the `style.fill` keyword. [https://github.com/terrastruct/d2/pull/452](https://github.com/terrastruct/d2/pull/452)
- Add official Docker image. See [./docs/INSTALL.md#docker](./docs/INSTALL.md#docker). [#76](https://github.com/terrastruct/d2/issues/76)
- Add `.msi` installer for convenient installation on Windows. [#379](https://github.com/terrastruct/d2/issues/379)
#### Improvements 🧹
- Fmt now preserves leading comment spacing.
- `d2 fmt` now preserves leading comment spacing.
[#400](https://github.com/terrastruct/d2/issues/400)
#### Bugfixes ⛑️
- Fixed crash when sequence diagrams had no messages.
[https://github.com/terrastruct/d2/pull/427](https://github.com/terrastruct/d2/pull/427)
- Fixed `constraint` keyword setting label.
[https://github.com/terrastruct/d2/issues/415](https://github.com/terrastruct/d2/issues/415)
- Fixed serialization affecting binary plugins (TALA).
[https://github.com/terrastruct/d2/pull/426](https://github.com/terrastruct/d2/pull/426)
- Fixed crash when sequence diagrams had no messages. [https://github.com/terrastruct/d2/pull/427](https://github.com/terrastruct/d2/pull/427)
- Fixed `constraint` keyword setting label. [https://github.com/terrastruct/d2/issues/415](https://github.com/terrastruct/d2/issues/415)
- Fixed serialization affecting binary plugins (TALA). [https://github.com/terrastruct/d2/pull/426](https://github.com/terrastruct/d2/pull/426)
- Fixed a connection rendering bug that could happen in firefox when there were no connection labels. [https://github.com/terrastruct/d2/pull/453](https://github.com/terrastruct/d2/pull/453)

View file

@ -8,5 +8,5 @@ RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.tar.gz" >/tmp/go.tar.gz
RUN tar -C /usr/local -xzf /tmp/go.tar.gz
ENV PATH="/usr/local/go/bin:$PATH"
RUN apt-get install -y build-essential
RUN apt-get install -y rsync
RUN apt-get install -y build-essential \
rsync

View file

@ -273,7 +273,7 @@ header() {
bigheader() {
set -- "$(echo "$*" | sed "s/^/ * /")"
FGCOLOR=${FGCOLOR:-3} logp "/****************************************************************
FGCOLOR=${FGCOLOR:-6} logp "/****************************************************************
$*
****************************************************************/"
}

1
ci/release/windows/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
d2.exe

BIN
ci/release/windows/d2.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 95 KiB

BIN
ci/release/windows/d2.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

39
ci/release/windows/d2.wxs Normal file
View file

@ -0,0 +1,39 @@
<!--
Wix documentation is nonexistent for v4. What exists is largely out of date and inconsistent.
This file was pieced together from:
1. https://www.firegiant.com/wix/tutorial/getting-started/
- This is for v3, I used wix convert to convert to v4
2. https://wixtoolset.org/docs/reference/schema/wxs/
3. Googling with trial and error
-->
<Wix xmlns="http://wixtoolset.org/schemas/v4/wxs">
<Package Name="D2" UpgradeCode="ac84fee7-eb67-4f5d-a08d-adef69538690" Language="1033" Codepage="utf-8" Version="$(var.D2Version)" Manufacturer="Terrastruct, Inc." InstallerVersion="200">
<SummaryInformation Keywords="Installer" Description="The D2 Installer" Manufacturer="Terrastruct, Inc." Codepage="1252" />
<Icon Id="d2.ico" SourceFile="d2.ico" />
<Property Id="ARPPRODUCTICON" Value="d2.ico" />
<Media Id="1" Cabinet="D2.cab" EmbedCab="yes" />
<Feature Id="Complete" Level="1">
<ComponentRef Id="Executable" />
</Feature>
<MajorUpgrade AllowSameVersionUpgrades='yes' DowngradeErrorMessage="A later version of [ProductName] is already installed. Setup will now exit."/>
<StandardDirectory Id="ProgramFiles64Folder">
<Directory Id="INSTALLDIR" Name="D2">
<Component Id="Executable" Guid="1090d036-c985-461f-94f6-3121dbcfcb48">
<File Id="D2EXE" Name="d2.exe" Source="d2.exe" KeyPath="yes" />
<Environment
Id="D2PathEntry"
Action="set"
Part="last"
Name="PATH"
Permanent="no"
System="yes"
Value="[INSTALLDIR]" />
</Component>
</Directory>
</StandardDirectory>
</Package>
</Wix>

2
ci/sub

@ -1 +1 @@
Subproject commit 7923b353a829cf289171baff0d0d017ae3d32278
Subproject commit 55bf1d39f3d524804422d4bdb4559cd6ff151b49

View file

@ -14,8 +14,9 @@ You may install `d2` through any of the following methods.
- <a href="#from-source" id="toc-from-source">From source</a>
- <a href="#source-release" id="toc-source-release">Source Release</a>
- <a href="#windows" id="toc-windows">Windows</a>
- <a href="#msys2" id="toc-msys2">MSYS2</a>
- <a href="#release-archives" id="toc-release-archives">Release archives</a>
- <a href="#wsl" id="toc-wsl">WSL</a>
- <a href="#docker" id="toc-docker">Docker</a>
- <a href="#coming-soon" id="toc-coming-soon">Coming soon</a>
## install.sh
@ -174,31 +175,19 @@ binary.
## Windows
d2 builds and runs on Windows:
We have prebuilt releases of d2 available for Windows via `.msi` installers. The installer
will add the `d2` binary to your `$PATH` so that you can execute `d2` in `cmd.exe` or
`pwsh.exe`.
We have prebuilt standalone releases for Windows though they're structured in the same way
as our Unix releases.
### Release archives
Easiest way to use d2 on Windows is to just `chdir` into the bin directory of the release
and invoke d2 like `./d2 <full-input-file-path>`
For installation, you'll have to put the `bin/d2.exe` binary into your `$PATH` or add the
`bin` directory of the release into your `$PATH`.
See https://www.wikihow.com/Change-the-PATH-Environment-Variable-on-Windows
Then you'll be able to call `d2` from the commandline in `cmd.exe` or `pwsh.exe`.
We intend to have a `.msi` release installer sometime soon that handles putting `d2` into
your `$PATH` for you.
### MSYS2
We also have release archives for Windows structured in the same way as our Unix releases
for use with MSYS2.
<img width="1680" alt="Screenshot 2022-12-06 at 2 55 27 AM" src="https://user-images.githubusercontent.com/10180857/205892927-6f3e116c-1c4a-440a-9972-82c306aa9779.png">
We recommend using [MSYS2](https://www.msys2.org/) or [Git
Bash](https://gitforwindows.org/#bash) (Git Bash is based on MSYS2) for an improved
terminal experience.
See [MSYS2](https://www.msys2.org/) or [Git Bash](https://gitforwindows.org/#bash) (Git
Bash is based on MSYS2).
MSYS2 provides a unix style shell environment that is native to Windows (unlike
[Cygwin](https://www.cygwin.com/)). MSYS2 allows `install.sh` to work, enables automatic
@ -216,9 +205,23 @@ under plain Windows.
aka Windows Subsystem for Linux if that's what you prefer. Installation is just like any
other Linux system.
## Docker
https://hub.docker.com/repository/docker/terrastruct/d2
We publish `amd64` and `arm64` images based on `debian:latest` for each release.
Example usage:
```sh
echo 'x -> y' >helloworld.d2
docker run --rm -it -u "$(id -u):$(id -g)" -v "$PWD:/root/src" \
-p 127.0.0.1:8080:8080 terrastruct/d2 --watch helloworld.d2
# Visit http://127.0.0.1:8080
```
## Coming soon
- Docker image
- rpm and deb packages
- with repositories and standalone
- homebrew core

View file

@ -278,7 +278,7 @@ header() {
bigheader() {
set -- "$(echo "$*" | sed "s/^/ * /")"
FGCOLOR=${FGCOLOR:-3} logp "/****************************************************************
FGCOLOR=${FGCOLOR:-6} logp "/****************************************************************
$*
****************************************************************/"
}