From d13f4a8935da900bdcf6ec3cb4211bf4fcd3a36e Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Mon, 14 Nov 2022 06:57:01 -0800 Subject: [PATCH] ci/release: More fixes --- ci/release/build.sh | 38 +++++++++++-- ci/release/builders/ensure_aws.sh | 95 ++++++++++++++++++++++++++++--- ci/sub | 2 +- 3 files changed, 121 insertions(+), 14 deletions(-) diff --git a/ci/release/build.sh b/ci/release/build.sh index 8aaec7c93..fed37c716 100755 --- a/ci/release/build.sh +++ b/ci/release/build.sh @@ -115,18 +115,30 @@ build() { fi case $OS in - # macos) - # ;; + macos) + case $ARCH in + amd64) + RHOST=$TSTRUCT_MACOS_AMD64_BUILDER build_rhost_macos + ;; + arm64) + RHOST=$TSTRUCT_MACOS_ARM64_BUILDER build_rhost_macos + ;; + *) + warn "no builder for OS=$OS ARCH=$ARCH, building locally..." + build_local + ;; + esac + ;; linux) case $ARCH in amd64) - RHOST=$TSTRUCT_LINUX_AMD64_BUILDER build_rhost + RHOST=$TSTRUCT_LINUX_AMD64_BUILDER build_rhost_linux ;; arm64) - RHOST=$TSTRUCT_LINUX_ARM64_BUILDER build_rhost + RHOST=$TSTRUCT_LINUX_ARM64_BUILDER build_rhost_linux ;; *) - warn "no builder for OS=$OS, building locally..." + warn "no builder for OS=$OS ARCH=$ARCH, building locally..." build_local ;; esac @@ -148,7 +160,21 @@ build_local() { sh_c ./ci/release/_build.sh } -build_rhost() { +build_rhost_macos() { + sh_c ssh "$RHOST" mkdir -p src + sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/" + sh_c ssh -tttt "$RHOST" "DRY_RUN=${DRY_RUN-} \ +HW_BUILD_DIR=$HW_BUILD_DIR \ +VERSION=$VERSION \ +OS=$OS \ +ARCH=$ARCH \ +ARCHIVE=$ARCHIVE \ +TERM=$TERM \ +./src/d2/ci/release/_build.sh" + sh_c rsync --archive --human-readable "$RHOST:src/d2/$ARCHIVE" "$ARCHIVE" +} + +build_rhost_linux() { sh_c ssh "$RHOST" mkdir -p src sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/" sh_c ssh -tttt "$RHOST" "DRY_RUN=${DRY_RUN-} \ diff --git a/ci/release/builders/ensure_aws.sh b/ci/release/builders/ensure_aws.sh index 9e4c35a53..3fa4da5c3 100755 --- a/ci/release/builders/ensure_aws.sh +++ b/ci/release/builders/ensure_aws.sh @@ -75,8 +75,8 @@ create_rhosts() { --instance-type=t2.small \ --security-groups=ssh \ "--key-name=$KEY_NAME" \ - --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' \ - 'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' >/dev/null + --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' \ + '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]"' >/dev/null fi while true; do dnsname=$(sh_c aws ec2 describe-instances \ @@ -100,8 +100,8 @@ create_rhosts() { --instance-type=t4g.small \ --security-groups=ssh \ "--key-name=$KEY_NAME" \ - --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' \ - 'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' >/dev/null + --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' \ + '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]"' >/dev/null fi while true; do dnsname=$(sh_c aws ec2 describe-instances \ @@ -114,18 +114,86 @@ create_rhosts() { fi sleep 5 done + + header "allocate macOS hosts" + if ! aws ec2 describe-hosts | grep -q mac1.metal; then + aws ec2 allocate-hosts --instance-type mac1.metal --quantity 1 --availability-zone us-west-2a + fi + if ! aws ec2 describe-hosts | grep -q mac2.metal; then + aws ec2 allocate-hosts --instance-type mac2.metal --quantity 1 --availability-zone us-west-2a + fi + + header macos-amd64 + if ! aws ec2 describe-instances \ + "--query=Reservations[*].Instances[?State.Name!='terminated']" \ + | grep -q d2-builder-macos-amd64; then + sh_c aws ec2 run-instances \ + --image-id=ami-0dd2ded7568750663 \ + --count=1 \ + --instance-type=mac1.metal \ + --security-groups=ssh \ + "--key-name=$KEY_NAME" \ + --placement 'Tenancy=host,HostId=h-0d881926de2e17555' \ + --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' \ + '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-amd64}]"' >/dev/null + fi + while true; do + dnsname=$(sh_c aws ec2 describe-instances \ + --filters 'Name=instance-state-name,Values=pending,running,shutting-down,stopping,stopped' 'Name=tag:Name,Values=d2-builder-macos-amd64' \ + | jq -r '.Reservations[].Instances[].PublicDnsName') + if [ -n "$dnsname" ]; then + log "TSTRUCT_MACOS_AMD64_BUILDER=ec2-user@$dnsname" + export TSTRUCT_MACOS_AMD64_BUILDER=ec2-user@$dnsname + break + fi + sleep 5 + done + + header macos-arm64 + if ! aws ec2 describe-instances \ + "--query=Reservations[*].Instances[?State.Name!='terminated']" \ + | grep -q d2-builder-macos-arm64; then + sh_c aws ec2 run-instances \ + --image-id=ami-0af0516ff2c43dbbe \ + --count=1 \ + --instance-type=mac2.metal \ + --security-groups=ssh \ + "--key-name=$KEY_NAME" \ + --placement 'Tenancy=host,HostId=h-040b9305c2d64119d' \ + --tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' \ + '"ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-macos-arm64}]"' >/dev/null + fi + while true; do + dnsname=$(sh_c aws ec2 describe-instances \ + --filters 'Name=instance-state-name,Values=pending,running,shutting-down,stopping,stopped' 'Name=tag:Name,Values=d2-builder-macos-arm64' \ + | jq -r '.Reservations[].Instances[].PublicDnsName') + if [ -n "$dnsname" ]; then + log "TSTRUCT_MACOS_ARM64_BUILDER=ec2-user@$dnsname" + export TSTRUCT_MACOS_ARM64_BUILDER=ec2-user@$dnsname + break + fi + sleep 5 + done } init_rhosts() { header linux-amd64 - RHOST=$TSTRUCT_LINUX_AMD64_BUILDER init_rhost + RHOST=$TSTRUCT_LINUX_AMD64_BUILDER init_rhost_linux header linux-arm64 - RHOST=$TSTRUCT_LINUX_ARM64_BUILDER init_rhost + RHOST=$TSTRUCT_LINUX_ARM64_BUILDER init_rhost_linux + header macos-amd64 + RHOST=$TSTRUCT_MACOS_AMD64_BUILDER init_rhost_macos + header macos-arm64 + RHOST=$TSTRUCT_MACOS_ARM64_BUILDER init_rhost_macos + + COLOR=2 header summary log "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER" log "export TSTRUCT_LINUX_ARM64_BUILDER=$TSTRUCT_LINUX_ARM64_BUILDER" + log "export TSTRUCT_MACOS_AMD64_BUILDER=$TSTRUCT_MACOS_AMD64_BUILDER" + log "export TSTRUCT_MACOS_ARM64_BUILDER=$TSTRUCT_MACOS_ARM64_BUILDER" } -init_rhost() { +init_rhost_linux() { while true; do if sh_c ssh "$RHOST" :; then break @@ -140,4 +208,17 @@ init_rhost() { sh_c ssh "$RHOST" 'sudo reboot' || true } +init_rhost_macos() { + while true; do + if sh_c ssh "$RHOST" :; then + break + fi + sleep 5 + done + sh_c ssh "$RHOST" '": | /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\""' + sh_c ssh "$RHOST" 'brew update' + sh_c ssh "$RHOST" 'brew upgrade' + sh_c ssh "$RHOST" 'brew install go' +} + main "$@" diff --git a/ci/sub b/ci/sub index 231dec909..be28ce747 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 231dec9096304f1a033fd2ef35228d4c6e0744b3 +Subproject commit be28ce747b280c59b8fe89e6b29bb0637984985d