ci: Prep for windows host

2022-12-14 00:47:07 -08:00 · 2022-12-14 00:47:07 -08:00 · d23eb5b332
commit d23eb5b332
parent 6e61645d0f
8 changed files with 56 additions and 98 deletions
--- a/ci/release/README.md
+++ b/ci/release/README.md
@ -23,12 +23,12 @@ it depends on from ../sub/lib.
 > variables as we must compile d2 directly on each release target to include dagre.
 > See https://github.com/terrastruct/d2/issues/31

-Use `--host-only` to build only the release for the host's OS-ARCH pair.
+Use `--host-only` to build only the release for the host's `$OS-$ARCH` pair.

 ### build_docker.sh

 Helper script called by build.sh to build D2 on each linux runner inside Docker.
-The Dockerfile is in ./builders/Dockerfile
+The Dockerfile is in ./linux/Dockerfile

 ### _build.sh

--- a/ci/release/builders/aws/ensure.sh
+++ b/ci/release/builders/aws/ensure.sh
@ -1,11 +1,11 @@
 #!/bin/sh
 set -eu
-cd -- "$(dirname "$0")/../../.."
-. ./ci/sub/lib.sh
+. "$(dirname "$0")/../../../../ci/sub/lib.sh"
+cd -- "$(dirname "$0")/../../../.."

 help() {
  cat <<EOF
-usage: $0 [--dry-run] [--skip-create]
+usage: $0 [--dry-run] [--skip-create] [--copy-id=id.pub]

 $0 creates and ensures the d2 builders in AWS.
 EOF
@ -18,14 +18,23 @@ main() {
        help
        return 0
        ;;
+      x)
+        flag_noarg && shift "$FLAGSHIFT"
+        set -x
+        export TRACE=1
+        ;;
      dry-run)
        flag_noarg && shift "$FLAGSHIFT"
-        DRY_RUN=1
+        export DRY_RUN=1
        ;;
      skip-create)
        flag_noarg && shift "$FLAGSHIFT"
        SKIP_CREATE=1
        ;;
+      copy-id)
+        flag_nonemptyarg && shift "$FLAGSHIFT"
+        ID_PUB_PATH=$FLAGARG
+        ;;
      *)
        flag_errusage "unrecognized flag $FLAGRAW"
        ;;
@ -35,6 +44,9 @@ main() {
  if [ $# -gt 0 ]; then
    flag_errusage "no arguments are accepted"
  fi
+  if [ -z "${ID_PUB_PATH-}" ]; then
+    flag_errusage "--copy-id is required"
+  fi

  if [ -z "${SKIP_CREATE-}" ]; then
    create_remote_hosts
@ -43,6 +55,8 @@ main() {
 }

 create_remote_hosts() {
+  bigheader create_remote_hosts
+
  KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName)
  VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId)

@ -73,11 +87,12 @@ create_remote_hosts() {
    | jq -r '.Reservations[].Instances[].State.Name')
  if [ -z "$state" ]; then
    sh_c aws ec2 run-instances \
-      --image-id=ami-071e6cafc48327ca2 \
+      --image-id=ami-0ecc74eca1d66d8a6 \
      --count=1 \
      --instance-type=t2.small \
      --security-groups=ssh \
      "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
      --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' \
        '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' >/dev/null
  fi
@ -86,8 +101,8 @@ create_remote_hosts() {
      --filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-amd64' \
      | jq -r '.Reservations[].Instances[].PublicDnsName')
    if [ -n "$dnsname" ]; then
-      log "TSTRUCT_LINUX_AMD64_BUILDER=admin@$dnsname"
-      export TSTRUCT_LINUX_AMD64_BUILDER=admin@$dnsname
+      log "TSTRUCT_LINUX_AMD64_BUILDER=ubuntu@$dnsname"
+      export TSTRUCT_LINUX_AMD64_BUILDER=ubuntu@$dnsname
      break
    fi
    sleep 5
@ -99,11 +114,12 @@ create_remote_hosts() {
    | jq -r '.Reservations[].Instances[].State.Name')
  if [ -z "$state" ]; then
    sh_c aws ec2 run-instances \
-      --image-id=ami-0e67506f183e5ab60 \
+      --image-id=ami-06e2dea2cdda3acda \
      --count=1 \
      --instance-type=t4g.small \
      --security-groups=ssh \
      "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
      --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' \
        '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' >/dev/null
  fi
@ -112,8 +128,8 @@ create_remote_hosts() {
      --filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=d2-builder-linux-arm64' \
      | jq -r '.Reservations[].Instances[].PublicDnsName')
    if [ -n "$dnsname" ]; then
-      log "TSTRUCT_LINUX_ARM64_BUILDER=admin@$dnsname"
-      export TSTRUCT_LINUX_ARM64_BUILDER=admin@$dnsname
+      log "TSTRUCT_LINUX_ARM64_BUILDER=ubuntu@$dnsname"
+      export TSTRUCT_LINUX_ARM64_BUILDER=ubuntu@$dnsname
      break
    fi
    sleep 5
@ -146,6 +162,7 @@ create_remote_hosts() {
      --instance-type=mac1.metal \
      --security-groups=ssh \
      "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
      --placement "Tenancy=host,HostId=$MACOS_AMD64_HOST_ID" \
      --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' \
        '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' >/dev/null
@ -173,6 +190,7 @@ create_remote_hosts() {
      --instance-type=mac2.metal \
      --security-groups=ssh \
      "--key-name=$KEY_NAME" \
+      --iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
      --placement "Tenancy=host,HostId=$MACOS_ARM64_HOST_ID" \
      --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' \
        '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' >/dev/null
@ -191,6 +209,8 @@ create_remote_hosts() {
 }

 init_remote_hosts() {
+  bigheader init_remote_hosts
+
  header linux-amd64
  REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER init_remote_linux
  header linux-arm64
@ -201,15 +221,19 @@ init_remote_hosts() {
  REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER init_remote_macos

  FGCOLOR=2 header summary
-  log "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER"
-  log "export TSTRUCT_LINUX_ARM64_BUILDER=$TSTRUCT_LINUX_ARM64_BUILDER"
-  log "export TSTRUCT_MACOS_AMD64_BUILDER=$TSTRUCT_MACOS_AMD64_BUILDER"
-  log "export TSTRUCT_MACOS_ARM64_BUILDER=$TSTRUCT_MACOS_ARM64_BUILDER"
+  echo "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER"
+  echo "export TSTRUCT_LINUX_ARM64_BUILDER=$TSTRUCT_LINUX_ARM64_BUILDER"
+  echo "export TSTRUCT_MACOS_AMD64_BUILDER=$TSTRUCT_MACOS_AMD64_BUILDER"
+  echo "export TSTRUCT_MACOS_ARM64_BUILDER=$TSTRUCT_MACOS_ARM64_BUILDER"
 }

 init_remote_linux() {
  wait_remote_host

+  if [ -n "${ID_PUB_PATH-}" ]; then
+    sh_c ssh_copy_id -i="$ID_PUB_PATH" "$REMOTE_HOST"
+  fi
+
  sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
 set -eux
 export DEBIAN_FRONTEND=noninteractive
@ -218,16 +242,16 @@ sudo -E apt-get update -y
 sudo -E apt-get dist-upgrade -y
 sudo -E apt-get install -y build-essential rsync

-# Docker from https://docs.docker.com/engine/install/debian/
+# Docker from https://docs.docker.com/engine/install/ubuntu/
 sudo -E apt-get -y install \
    ca-certificates \
    curl \
    gnupg \
    lsb-release
 sudo mkdir -p /etc/apt/keyrings
-curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --yes --dearmor -o /etc/apt/keyrings/docker.gpg
 echo \
-  "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+  "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 sudo -E apt-get update -y
 sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
@ -239,6 +263,11 @@ mkdir -p \$HOME/.local/share/man
 EOF
  init_remote_env

+  sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
+set -eux
+export DEBIAN_FRONTEND=noninteractive
+sudo -E apt-get autoremove -y
+EOF
  sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
 }

@ -266,6 +295,8 @@ EOF
  sh_c ssh "$REMOTE_HOST" brew update
  sh_c ssh "$REMOTE_HOST" brew upgrade
  sh_c ssh "$REMOTE_HOST" brew install go rsync
+
+  sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
 }

 init_remote_env() {
--- a/ci/release/builders/aws/ssh.sh
+++ b/ci/release/builders/aws/ssh.sh
@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
-cd -- "$(dirname "$0")/../../.."
-. ./ci/sub/lib.sh
+. "$(dirname "$0")/../../../../ci/sub/lib.sh"
+cd -- "$(dirname "$0")/../../../.."

 help() {
  cat <<EOF
--- a/ci/release/build_docker.sh
+++ b/ci/release/build_docker.sh
@ -5,7 +5,7 @@ cd -- "$(dirname "$0")/../.."

 tag="$(sh_c docker build \
  --build-arg GOVERSION="1.19.3.linux-$ARCH" \
-  -qf ./ci/release/builders/Dockerfile ./ci/release/builders)"
+  -qf ./ci/release/linux/Dockerfile ./ci/release/linux)"
 docker_run \
  -e DRY_RUN \
  -e HW_BUILD_DIR \
--- a/ci/release/builders/Dockerfile-centos
+++ b/ci/release/builders/Dockerfile-centos
@ -1,16 +0,0 @@
-FROM centos:7
-
-ARG GOVERSION=
-
-RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.tar.gz" >/tmp/go.tar.gz
-RUN tar -C /usr/local -xzf /tmp/go.tar.gz
-
-ENV PATH="/usr/local/go/bin:$PATH"
-
-RUN yum install -y rsync wget
-RUN yum groupinstall -y 'Development Tools'
-
-RUN curl -fsSL https://ftp.gnu.org/gnu/gcc/gcc-5.2.0/gcc-5.2.0.tar.gz >/tmp/gcc.tar.gz
-RUN tar -C /usr/local -xzf /tmp/gcc.tar.gz
-RUN cd /usr/local/gcc-5.2.0 && ./contrib/download_prerequisites && mkdir -p build \
-  && cd build && ../configure --disable-multilib && make && make install
--- a/ci/release/builders/aws/copy-id.sh
+++ b/ci/release/builders/aws/copy-id.sh
@ -1,57 +0,0 @@
-#!/bin/sh
-set -eu
-cd -- "$(dirname "$0")/../../.."
-. ./ci/sub/lib.sh
-
-help() {
-  cat <<EOF
-usage: $0 [--dry-run] -i keys.pub
-
-$0 copies keys.pub to each builder and then deduplicates its .authorized_keys.
-EOF
-}
-
-main() {
-  while flag_parse "$@"; do
-    case "$FLAG" in
-      h|help)
-        help
-        return 0
-        ;;
-      dry-run)
-        flag_noarg && shift "$FLAGSHIFT"
-        DRY_RUN=1
-        ;;
-      i)
-        flag_nonemptyarg && shift "$FLAGSHIFT"
-        KEY_FILE=$FLAGARG
-        ;;
-      *)
-        flag_errusage "unrecognized flag $FLAGRAW"
-        ;;
-    esac
-  done
-  shift "$FLAGSHIFT"
-  if [ -z "${KEY_FILE-}" ]; then
-    echoerr "-i is required"
-    exit 1
-  fi
-
-  header linux-amd64
-  REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER copy_keys
-  header linux-arm64
-  REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER copy_keys
-  header macos-amd64
-  REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER copy_keys
-  header macos-arm64
-  REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER copy_keys
-}
-
-copy_keys() {
-  sh_c ssh-copy-id -fi "$KEY_FILE" "$REMOTE_HOST"
-  sh_c ssh "$REMOTE_HOST" 'cat .ssh/authorized_keys \| sort -u \> .ssh/authorized_keys.dedup'
-  sh_c ssh "$REMOTE_HOST" 'cp .ssh/authorized_keys.dedup .ssh/authorized_keys'
-  sh_c ssh "$REMOTE_HOST" 'rm .ssh/authorized_keys.dedup'
-}
-
-main "$@"
--- a/ci/release/builders/Dockerfile
+++ b/ci/release/builders/Dockerfile
@ -8,5 +8,5 @@ RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.tar.gz" >/tmp/go.tar.gz
 RUN tar -C /usr/local -xzf /tmp/go.tar.gz
 ENV PATH="/usr/local/go/bin:$PATH"

-RUN apt-get install -y build-essential
-RUN apt-get install -y rsync
+RUN apt-get install -y build-essential \
+  rsync
--- a/ci/sub
+++ b/ci/sub
@ -1 +1 @@
-Subproject commit 7923b353a829cf289171baff0d0d017ae3d32278
+Subproject commit d103a19548c72d68d8aab26da1fda158489252ba