ci: Prep for windows host

ci/release/README.md

@@ -23,12 +23,12 @@ it depends on from ../sub/lib.
 > variables as we must compile d2 directly on each release target to include dagre.
 > See https://github.com/terrastruct/d2/issues/31
 
-Use `--host-only` to build only the release for the host's OS-ARCH pair.
+Use `--host-only` to build only the release for the host's `$OS-$ARCH` pair.
 
 ### build_docker.sh
 
 Helper script called by build.sh to build D2 on each linux runner inside Docker.
-The Dockerfile is in ./builders/Dockerfile
+The Dockerfile is in ./linux/Dockerfile
 
 ### _build.sh
 

ci/release/aws/ensure.sh

@@ -1,11 +1,11 @@
 #!/bin/sh
 set -eu
-cd -- "$(dirname "$0")/../../.."
+. "$(dirname "$0")/../../../../ci/sub/lib.sh"
-. ./ci/sub/lib.sh
+cd -- "$(dirname "$0")/../../../.."
 
 help() {
   cat <<EOF
-usage: $0 [--dry-run] [--skip-create]
+usage: $0 [--dry-run] [--skip-create] [--copy-id=id.pub]
 
 $0 creates and ensures the d2 builders in AWS.
 EOF
@@ -18,14 +18,23 @@ main() {
         help
         return 0
         ;;
+      x)
+        flag_noarg && shift "$FLAGSHIFT"
+        set -x
+        export TRACE=1
+        ;;
       dry-run)
         flag_noarg && shift "$FLAGSHIFT"
-        DRY_RUN=1
+        export DRY_RUN=1
         ;;
       skip-create)
         flag_noarg && shift "$FLAGSHIFT"
         SKIP_CREATE=1
         ;;
+      copy-id)
+        flag_nonemptyarg && shift "$FLAGSHIFT"
+        ID_PUB_PATH=$FLAGARG
+        ;;
       *)
         flag_errusage "unrecognized flag $FLAGRAW"
         ;;
@@ -35,6 +44,9 @@ main() {
   if [ $# -gt 0 ]; then
     flag_errusage "no arguments are accepted"
   fi
+  if [ -z "${ID_PUB_PATH-}" ]; then
+    flag_errusage "--copy-id is required"
+  fi
 
   if [ -z "${SKIP_CREATE-}" ]; then
     create_remote_hosts
@@ -43,6 +55,8 @@ main() {
 }
 
 create_remote_hosts() {
+  bigheader create_remote_hosts
+
   KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName)
   VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId)
 
@@ -73,11 +87,12 @@ create_remote_hosts() {
     | jq -r '.Reservations[].Instances[].State.Name')
   if [ -z "$state" ]; then
     sh_c aws ec2 run-instances \
-      --image-id=ami-071e6cafc48327ca2 \
+      --image-id=ami-0ecc74eca1d66d8a6 \
       --count=1 \
       --instance-type=t2.small \
       --security-groups=ssh \
       "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
       --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' \
         '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' >/dev/null
   fi
@@ -86,8 +101,8 @@ create_remote_hosts() {
       --filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-amd64' \
       | jq -r '.Reservations[].Instances[].PublicDnsName')
     if [ -n "$dnsname" ]; then
-      log "TSTRUCT_LINUX_AMD64_BUILDER=admin@$dnsname"
+      log "TSTRUCT_LINUX_AMD64_BUILDER=ubuntu@$dnsname"
-      export TSTRUCT_LINUX_AMD64_BUILDER=admin@$dnsname
+      export TSTRUCT_LINUX_AMD64_BUILDER=ubuntu@$dnsname
       break
     fi
     sleep 5
@@ -99,11 +114,12 @@ create_remote_hosts() {
     | jq -r '.Reservations[].Instances[].State.Name')
   if [ -z "$state" ]; then
     sh_c aws ec2 run-instances \
-      --image-id=ami-0e67506f183e5ab60 \
+      --image-id=ami-06e2dea2cdda3acda \
       --count=1 \
       --instance-type=t4g.small \
       --security-groups=ssh \
       "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
       --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' \
         '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' >/dev/null
   fi
@@ -112,8 +128,8 @@ create_remote_hosts() {
       --filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-arm64' \
       | jq -r '.Reservations[].Instances[].PublicDnsName')
     if [ -n "$dnsname" ]; then
-      log "TSTRUCT_LINUX_ARM64_BUILDER=admin@$dnsname"
+      log "TSTRUCT_LINUX_ARM64_BUILDER=ubuntu@$dnsname"
-      export TSTRUCT_LINUX_ARM64_BUILDER=admin@$dnsname
+      export TSTRUCT_LINUX_ARM64_BUILDER=ubuntu@$dnsname
       break
     fi
     sleep 5
@@ -146,6 +162,7 @@ create_remote_hosts() {
       --instance-type=mac1.metal \
       --security-groups=ssh \
       "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
       --placement "Tenancy=host,HostId=$MACOS_AMD64_HOST_ID" \
       --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' \
         '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' >/dev/null
@@ -173,6 +190,7 @@ create_remote_hosts() {
       --instance-type=mac2.metal \
       --security-groups=ssh \
       "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
       --placement "Tenancy=host,HostId=$MACOS_ARM64_HOST_ID" \
       --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' \
         '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' >/dev/null
@@ -191,6 +209,8 @@ create_remote_hosts() {
 }
 
 init_remote_hosts() {
+  bigheader init_remote_hosts
+
   header linux-amd64
   REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER init_remote_linux
   header linux-arm64
@@ -201,15 +221,19 @@ init_remote_hosts() {
   REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER init_remote_macos
 
   FGCOLOR=2 header summary
-  log "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER"
+  echo "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER"
-  log "export TSTRUCT_LINUX_ARM64_BUILDER=$TSTRUCT_LINUX_ARM64_BUILDER"
+  echo "export TSTRUCT_LINUX_ARM64_BUILDER=$TSTRUCT_LINUX_ARM64_BUILDER"
-  log "export TSTRUCT_MACOS_AMD64_BUILDER=$TSTRUCT_MACOS_AMD64_BUILDER"
+  echo "export TSTRUCT_MACOS_AMD64_BUILDER=$TSTRUCT_MACOS_AMD64_BUILDER"
-  log "export TSTRUCT_MACOS_ARM64_BUILDER=$TSTRUCT_MACOS_ARM64_BUILDER"
+  echo "export TSTRUCT_MACOS_ARM64_BUILDER=$TSTRUCT_MACOS_ARM64_BUILDER"
 }
 
 init_remote_linux() {
   wait_remote_host
 
+  if [ -n "${ID_PUB_PATH-}" ]; then
+    sh_c ssh_copy_id -i="$ID_PUB_PATH" "$REMOTE_HOST"
+  fi
+
   sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
 set -eux
 export DEBIAN_FRONTEND=noninteractive
@@ -218,16 +242,16 @@ sudo -E apt-get update -y
 sudo -E apt-get dist-upgrade -y
 sudo -E apt-get install -y build-essential rsync
 
-# Docker from https://docs.docker.com/engine/install/debian/
+# Docker from https://docs.docker.com/engine/install/ubuntu/
 sudo -E apt-get -y install \
     ca-certificates \
     curl \
     gnupg \
     lsb-release
 sudo mkdir -p /etc/apt/keyrings
-curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --yes --dearmor -o /etc/apt/keyrings/docker.gpg
 echo \
-  "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+  "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
   \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 sudo -E apt-get update -y
 sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
@@ -239,6 +263,11 @@ mkdir -p \$HOME/.local/share/man
 EOF
   init_remote_env
 
+  sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
+set -eux
+export DEBIAN_FRONTEND=noninteractive
+sudo -E apt-get autoremove -y
+EOF
   sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
 }
 
@@ -266,6 +295,8 @@ EOF
   sh_c ssh "$REMOTE_HOST" brew update
   sh_c ssh "$REMOTE_HOST" brew upgrade
   sh_c ssh "$REMOTE_HOST" brew install go rsync
+
+  sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
 }
 
 init_remote_env() {

ci/release/aws/ssh.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
-cd -- "$(dirname "$0")/../../.."
+. "$(dirname "$0")/../../../../ci/sub/lib.sh"
-. ./ci/sub/lib.sh
+cd -- "$(dirname "$0")/../../../.."
 
 help() {
   cat <<EOF

ci/release/build_docker.sh

@@ -5,7 +5,7 @@ cd -- "$(dirname "$0")/../.."
 
 tag="$(sh_c docker build \
   --build-arg GOVERSION="1.19.3.linux-$ARCH" \
-  -qf ./ci/release/builders/Dockerfile ./ci/release/builders)"
+  -qf ./ci/release/linux/Dockerfile ./ci/release/linux)"
 docker_run \
   -e DRY_RUN \
   -e HW_BUILD_DIR \

ci/release/builders/Dockerfile-centos

@@ -1,16 +0,0 @@
-FROM centos:7
-
-ARG GOVERSION=
-
-RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.tar.gz" >/tmp/go.tar.gz
-RUN tar -C /usr/local -xzf /tmp/go.tar.gz
-
-ENV PATH="/usr/local/go/bin:$PATH"
-
-RUN yum install -y rsync wget
-RUN yum groupinstall -y 'Development Tools'
-
-RUN curl -fsSL https://ftp.gnu.org/gnu/gcc/gcc-5.2.0/gcc-5.2.0.tar.gz >/tmp/gcc.tar.gz
-RUN tar -C /usr/local -xzf /tmp/gcc.tar.gz
-RUN cd /usr/local/gcc-5.2.0 && ./contrib/download_prerequisites && mkdir -p build \
-  && cd build && ../configure --disable-multilib && make && make install

ci/release/builders/aws/copy-id.sh

@@ -1,57 +0,0 @@
-#!/bin/sh
-set -eu
-cd -- "$(dirname "$0")/../../.."
-. ./ci/sub/lib.sh
-
-help() {
-  cat <<EOF
-usage: $0 [--dry-run] -i keys.pub
-
-$0 copies keys.pub to each builder and then deduplicates its .authorized_keys.
-EOF
-}
-
-main() {
-  while flag_parse "$@"; do
-    case "$FLAG" in
-      h|help)
-        help
-        return 0
-        ;;
-      dry-run)
-        flag_noarg && shift "$FLAGSHIFT"
-        DRY_RUN=1
-        ;;
-      i)
-        flag_nonemptyarg && shift "$FLAGSHIFT"
-        KEY_FILE=$FLAGARG
-        ;;
-      *)
-        flag_errusage "unrecognized flag $FLAGRAW"
-        ;;
-    esac
-  done
-  shift "$FLAGSHIFT"
-  if [ -z "${KEY_FILE-}" ]; then
-    echoerr "-i is required"
-    exit 1
-  fi
-
-  header linux-amd64
-  REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER copy_keys
-  header linux-arm64
-  REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER copy_keys
-  header macos-amd64
-  REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER copy_keys
-  header macos-arm64
-  REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER copy_keys
-}
-
-copy_keys() {
-  sh_c ssh-copy-id -fi "$KEY_FILE" "$REMOTE_HOST"
-  sh_c ssh "$REMOTE_HOST" 'cat .ssh/authorized_keys \| sort -u \> .ssh/authorized_keys.dedup'
-  sh_c ssh "$REMOTE_HOST" 'cp .ssh/authorized_keys.dedup .ssh/authorized_keys'
-  sh_c ssh "$REMOTE_HOST" 'rm .ssh/authorized_keys.dedup'
-}
-
-main "$@"

ci/release/linux/Dockerfile

@@ -8,5 +8,5 @@ RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.tar.gz" >/tmp/go.tar.gz
 RUN tar -C /usr/local -xzf /tmp/go.tar.gz
 ENV PATH="/usr/local/go/bin:$PATH"
 
-RUN apt-get install -y build-essential
+RUN apt-get install -y build-essential \
-RUN apt-get install -y rsync
+  rsync

ci/sub

@@ -1 +1 @@
-Subproject commit 7923b353a829cf289171baff0d0d017ae3d32278
+Subproject commit d103a19548c72d68d8aab26da1fda158489252ba