From 0af598557c6a4bd35a4c17a91aa53de070b103a6 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Tue, 15 Nov 2022 16:32:01 -0800 Subject: [PATCH 01/10] pull_request_template: Add reminder to update changelog and docs --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e61c2ae64..f24865554 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,2 +1,3 @@ + From 9fefb95525f5b954718954f832b8c8c116abba73 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Tue, 15 Nov 2022 17:49:37 -0800 Subject: [PATCH 02/10] ci/sub: Update [ci-base] --- ci/sub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sub b/ci/sub index 2fd92e497..237ad4b4b 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 2fd92e49721b984e3a3ce9ea64810947c5eaf442 +Subproject commit 237ad4b4bc52c0b27eb9d9b7e1496d8a9453dd05 From 756a2315d96922f0fc512ef45370933a88495f9d Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 03:16:57 -0800 Subject: [PATCH 03/10] ci: Fixes --- ci/release/template/scripts/install.sh | 4 ++-- ci/release/template/scripts/lib.sh | 4 ++++ ci/release/template/scripts/uninstall.sh | 2 +- ci/sub | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/release/template/scripts/install.sh b/ci/release/template/scripts/install.sh index 4b4ae2d08..043e5d098 100755 --- a/ci/release/template/scripts/install.sh +++ b/ci/release/template/scripts/install.sh @@ -4,14 +4,14 @@ cd -- "$(dirname "$0")/.." . ./scripts/lib.sh main() { - if [ ! -e "${PREFIX-}" ]; then + if [ -z "${PREFIX-}" ]; then echoerr "\$PREFIX must be set to a unix prefix directory in which to install d2 like /usr/local" return 1 fi sh_c mkdir -p "$PREFIX/bin" - sh_c mkdir -p "$PREFIX/share/man/man1" sh_c install ./bin/d2 "$PREFIX/bin/d2" + sh_c mkdir -p "$PREFIX/share/man/man1" sh_c install ./man/d2.1 "$PREFIX/share/man/man1" } diff --git a/ci/release/template/scripts/lib.sh b/ci/release/template/scripts/lib.sh index 95244284d..0a232eb74 100644 --- a/ci/release/template/scripts/lib.sh +++ b/ci/release/template/scripts/lib.sh @@ -1,5 +1,9 @@ #!/bin/sh +if [ -n "${DEBUG-}" ]; then + set -x +fi + rand() { seed="$1" range="$2" diff --git a/ci/release/template/scripts/uninstall.sh b/ci/release/template/scripts/uninstall.sh index d195463f3..e21877ca9 100755 --- a/ci/release/template/scripts/uninstall.sh +++ b/ci/release/template/scripts/uninstall.sh @@ -4,7 +4,7 @@ cd -- "$(dirname "$0")/.." . ./scripts/lib.sh main() { - if [ ! -e "${PREFIX-}" ]; then + if [ -z "${PREFIX-}" ]; then echoerr "\$PREFIX must be set to a unix prefix directory from which to uninstall d2 like /usr/local" return 1 fi diff --git a/ci/sub b/ci/sub index 237ad4b4b..75d882344 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 237ad4b4bc52c0b27eb9d9b7e1496d8a9453dd05 +Subproject commit 75d882344578aa409a8f81c8f1ee931e0f0297d0 From 93e95ad68f3f27be01302b24ddcb3cb8bc61d108 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 04:01:42 -0800 Subject: [PATCH 04/10] release/build.sh: Lock remote hosts when building --- ci/release/build.sh | 4 ++++ ci/sub | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/release/build.sh b/ci/release/build.sh index b716a91a0..9a5e6219d 100755 --- a/ci/release/build.sh +++ b/ci/release/build.sh @@ -161,6 +161,8 @@ build_local() { } build_rhost_macos() { + sh_c lockfile_ssh "$RHOST" .d2-build-lock + trap unlockfile_ssh EXIT sh_c ssh "$RHOST" mkdir -p src sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/" sh_c ssh -tttt "$RHOST" "DRY_RUN=${DRY_RUN-} \ @@ -177,6 +179,8 @@ PATH=\\\"/usr/local/bin:/usr/local/sbin:/opt/homebrew/bin:/opt/homebrew/sbin\\\$ } build_rhost_linux() { + sh_c lockfile_ssh "$RHOST" .d2-build-lock + trap unlockfile_ssh EXIT sh_c ssh "$RHOST" mkdir -p src sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/" sh_c ssh -tttt "$RHOST" "DRY_RUN=${DRY_RUN-} \ diff --git a/ci/sub b/ci/sub index 75d882344..9bc384b72 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 75d882344578aa409a8f81c8f1ee931e0f0297d0 +Subproject commit 9bc384b724ff8b9007da5ecec809d70f9959a018 From 4c731d7b2447f6b67205b46fbc22cf78e31737cd Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 04:02:35 -0800 Subject: [PATCH 05/10] release/build.sh: RHOST -> REMOTE_HOST --- ci/release/build.sh | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ci/release/build.sh b/ci/release/build.sh index 9a5e6219d..242ebd6de 100755 --- a/ci/release/build.sh +++ b/ci/release/build.sh @@ -118,10 +118,10 @@ build() { macos) case $ARCH in amd64) - RHOST=$TSTRUCT_MACOS_AMD64_BUILDER build_rhost_macos + REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER build_remote_macos ;; arm64) - RHOST=$TSTRUCT_MACOS_ARM64_BUILDER build_rhost_macos + REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER build_remote_macos ;; *) warn "no builder for OS=$OS ARCH=$ARCH, building locally..." @@ -132,10 +132,10 @@ build() { linux) case $ARCH in amd64) - RHOST=$TSTRUCT_LINUX_AMD64_BUILDER build_rhost_linux + REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER build_remote_linux ;; arm64) - RHOST=$TSTRUCT_LINUX_ARM64_BUILDER build_rhost_linux + REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER build_remote_linux ;; *) warn "no builder for OS=$OS ARCH=$ARCH, building locally..." @@ -160,12 +160,12 @@ build_local() { sh_c ./ci/release/_build.sh } -build_rhost_macos() { - sh_c lockfile_ssh "$RHOST" .d2-build-lock +build_remote_macos() { + sh_c lockfile_ssh "$REMOTE_HOST" .d2-build-lock trap unlockfile_ssh EXIT - sh_c ssh "$RHOST" mkdir -p src - sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/" - sh_c ssh -tttt "$RHOST" "DRY_RUN=${DRY_RUN-} \ + sh_c ssh "$REMOTE_HOST" mkdir -p src + sh_c rsync --archive --human-readable --delete ./ "$REMOTE_HOST:src/d2/" + sh_c ssh -tttt "$REMOTE_HOST" "DRY_RUN=${DRY_RUN-} \ HW_BUILD_DIR=$HW_BUILD_DIR \ VERSION=$VERSION \ OS=$OS \ @@ -175,15 +175,15 @@ TERM=$TERM \ PATH=\\\"/usr/local/bin:/usr/local/sbin:/opt/homebrew/bin:/opt/homebrew/sbin\\\${PATH+:\\\$PATH}\\\" \ ./src/d2/ci/release/_build.sh" sh_c mkdir -p "$HW_BUILD_DIR" - sh_c rsync --archive --human-readable "$RHOST:src/d2/$ARCHIVE" "$ARCHIVE" + sh_c rsync --archive --human-readable "$REMOTE_HOST:src/d2/$ARCHIVE" "$ARCHIVE" } -build_rhost_linux() { - sh_c lockfile_ssh "$RHOST" .d2-build-lock +build_remote_linux() { + sh_c lockfile_ssh "$REMOTE_HOST" .d2-build-lock trap unlockfile_ssh EXIT - sh_c ssh "$RHOST" mkdir -p src - sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/" - sh_c ssh -tttt "$RHOST" "DRY_RUN=${DRY_RUN-} \ + sh_c ssh "$REMOTE_HOST" mkdir -p src + sh_c rsync --archive --human-readable --delete ./ "$REMOTE_HOST:src/d2/" + sh_c ssh -tttt "$REMOTE_HOST" "DRY_RUN=${DRY_RUN-} \ HW_BUILD_DIR=$HW_BUILD_DIR \ VERSION=$VERSION \ OS=$OS \ @@ -192,7 +192,7 @@ ARCHIVE=$ARCHIVE \ TERM=$TERM \ ./src/d2/ci/release/build_docker.sh" sh_c mkdir -p "$HW_BUILD_DIR" - sh_c rsync --archive --human-readable "$RHOST:src/d2/$ARCHIVE" "$ARCHIVE" + sh_c rsync --archive --human-readable "$REMOTE_HOST:src/d2/$ARCHIVE" "$ARCHIVE" } ssh() { From 358862e317995e446bb32faffe4615be4cb706a6 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 04:24:51 -0800 Subject: [PATCH 06/10] ci/sub: Update --- ci/sub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sub b/ci/sub index 9bc384b72..7f058d87e 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 9bc384b724ff8b9007da5ecec809d70f9959a018 +Subproject commit 7f058d87e852cc099f51ee5d5056ebd6f87c3dd6 From f630edba2261bca48f667f3e2eb972f9b4a5f688 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 05:17:57 -0800 Subject: [PATCH 07/10] install.sh: Output MANPATH with trailing colon Bizarre requirement to extend the system MANPATH. --- ci/release/_install.sh | 2 +- install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/release/_install.sh b/ci/release/_install.sh index 7afd53f1e..7406eca3f 100755 --- a/ci/release/_install.sh +++ b/ci/release/_install.sh @@ -194,7 +194,7 @@ EOF if ! manpath | grep -qF "$PREFIX/share/man"; then logcat >&2 <&2 < Date: Wed, 16 Nov 2022 05:18:27 -0800 Subject: [PATCH 08/10] ci/sub: Update --- ci/sub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sub b/ci/sub index 7f058d87e..6267fa6de 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 7f058d87e852cc099f51ee5d5056ebd6f87c3dd6 +Subproject commit 6267fa6de949a1639435fb857154e15aab61f172 From 4647ca32643ce774b24ea450742fbe5b208c4ddf Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 05:28:37 -0800 Subject: [PATCH 09/10] ci/sub: Update --- ci/sub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sub b/ci/sub index 6267fa6de..65baeaa0e 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 6267fa6de949a1639435fb857154e15aab61f172 +Subproject commit 65baeaa0e48fff12b12b1d04a0b013ba71e20add From 61bf09d088d70dfc796c4ea5607a114bd284cbcf Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 06:14:07 -0800 Subject: [PATCH 10/10] aws: Update scripts --- ci/release/builders/aws_copy_keys.sh | 61 ++++++++++++++++++++++++++ ci/release/builders/aws_ensure.sh | 44 +++++++++---------- ci/release/builders/aws_ssh_copy_id.sh | 43 ------------------ 3 files changed, 83 insertions(+), 65 deletions(-) create mode 100755 ci/release/builders/aws_copy_keys.sh delete mode 100755 ci/release/builders/aws_ssh_copy_id.sh diff --git a/ci/release/builders/aws_copy_keys.sh b/ci/release/builders/aws_copy_keys.sh new file mode 100755 index 000000000..df5c46408 --- /dev/null +++ b/ci/release/builders/aws_copy_keys.sh @@ -0,0 +1,61 @@ +#!/bin/sh +set -eu +cd -- "$(dirname "$0")/../../.." +. ./ci/sub/lib.sh + +help() { + cat < .ssh/authorized_keys.dedup' + sh_c ssh "$REMOTE_HOST" 'cp .ssh/authorized_keys.dedup .ssh/authorized_keys' + sh_c ssh "$REMOTE_HOST" 'rm .ssh/authorized_keys.dedup' +} + +main "$@" diff --git a/ci/release/builders/aws_ensure.sh b/ci/release/builders/aws_ensure.sh index dfeb3443c..c82d68fdb 100755 --- a/ci/release/builders/aws_ensure.sh +++ b/ci/release/builders/aws_ensure.sh @@ -41,12 +41,12 @@ main() { fi if [ -z "${SKIP_CREATE-}" ]; then - create_rhosts + create_remote_hosts fi - init_rhosts + init_remote_hosts } -create_rhosts() { +create_remote_hosts() { KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName) VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId) @@ -194,15 +194,15 @@ create_rhosts() { done } -init_rhosts() { +init_remote_hosts() { header linux-amd64 - RHOST=$TSTRUCT_LINUX_AMD64_BUILDER init_rhost_linux + REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER init_remote_linux header linux-arm64 - RHOST=$TSTRUCT_LINUX_ARM64_BUILDER init_rhost_linux + REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER init_remote_linux header macos-amd64 - RHOST=$TSTRUCT_MACOS_AMD64_BUILDER init_rhost_macos + REMOTE_HOST=$TSTRUCT_MACOS_AMD64_BUILDER init_remote_macos header macos-arm64 - RHOST=$TSTRUCT_MACOS_ARM64_BUILDER init_rhost_macos + REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER init_remote_macos COLOR=2 header summary log "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER" @@ -211,32 +211,32 @@ init_rhosts() { log "export TSTRUCT_MACOS_ARM64_BUILDER=$TSTRUCT_MACOS_ARM64_BUILDER" } -init_rhost_linux() { +init_remote_linux() { while true; do - if sh_c ssh "$RHOST" :; then + if sh_c ssh "$REMOTE_HOST" :; then break fi sleep 5 done - sh_c ssh "$RHOST" 'sudo yum upgrade -y' - sh_c ssh "$RHOST" 'sudo yum install -y docker' - sh_c ssh "$RHOST" 'sudo systemctl start docker' - sh_c ssh "$RHOST" 'sudo systemctl enable docker' - sh_c ssh "$RHOST" 'sudo usermod -a -G docker ec2-user' - sh_c ssh "$RHOST" 'sudo reboot' || true + sh_c ssh "$REMOTE_HOST" 'sudo yum upgrade -y' + sh_c ssh "$REMOTE_HOST" 'sudo yum install -y docker' + sh_c ssh "$REMOTE_HOST" 'sudo systemctl start docker' + sh_c ssh "$REMOTE_HOST" 'sudo systemctl enable docker' + sh_c ssh "$REMOTE_HOST" 'sudo usermod -a -G docker ec2-user' + sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true } -init_rhost_macos() { +init_remote_macos() { while true; do - if sh_c ssh "$RHOST" :; then + if sh_c ssh "$REMOTE_HOST" :; then break fi sleep 5 done - sh_c ssh "$RHOST" '": | /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\""' - sh_c ssh "$RHOST" 'PATH="/usr/local/bin:/opt/homebrew/bin:\$PATH" brew update' - sh_c ssh "$RHOST" 'PATH="/usr/local/bin:/opt/homebrew/bin:\$PATH" brew upgrade' - sh_c ssh "$RHOST" 'PATH="/usr/local/bin:/opt/homebrew/bin:\$PATH" brew install go' + sh_c ssh "$REMOTE_HOST" '": | /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\""' + sh_c ssh "$REMOTE_HOST" 'PATH="/usr/local/bin:/opt/homebrew/bin:\$PATH" brew update' + sh_c ssh "$REMOTE_HOST" 'PATH="/usr/local/bin:/opt/homebrew/bin:\$PATH" brew upgrade' + sh_c ssh "$REMOTE_HOST" 'PATH="/usr/local/bin:/opt/homebrew/bin:\$PATH" brew install go' } main "$@" diff --git a/ci/release/builders/aws_ssh_copy_id.sh b/ci/release/builders/aws_ssh_copy_id.sh deleted file mode 100755 index d50a96d9d..000000000 --- a/ci/release/builders/aws_ssh_copy_id.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -set -eu -cd -- "$(dirname "$0")/../../.." -. ./ci/sub/lib.sh - -help() { - cat <