diff --git a/ci/release/_build.sh b/ci/release/_build.sh new file mode 100755 index 000000000..d30527b0c --- /dev/null +++ b/ci/release/_build.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd -- "$(dirname "$0")/../.." +. ./ci/sub/lib.sh + +sh_c mkdir -p "$HW_BUILD_DIR" +sh_c rsync --recursive --perms --delete \ + --human-readable --copy-links ./ci/release/template/ "$HW_BUILD_DIR/" +VERSION=$VERSION sh_c eval "'$HW_BUILD_DIR/README.md.sh'" \> "'$HW_BUILD_DIR/README.md'" +sh_c rm -f "$HW_BUILD_DIR/README.md.sh" +sh_c find "$HW_BUILD_DIR" -exec touch {} \; + +export GOOS=$(goos "$OS") +export GOARCH="$ARCH" +sh_c mkdir -p "$HW_BUILD_DIR/bin" +sh_c go build -ldflags "-X oss.terrastruct.com/d2/lib/version.Version=$VERSION" \ + -o "$HW_BUILD_DIR/bin/d2" ./cmd/d2 + +sh_c tar czf "$ARCHIVE" "$HW_BUILD_DIR" diff --git a/ci/release/build.sh b/ci/release/build.sh index 3ddbd47de..f944e08e7 100755 --- a/ci/release/build.sh +++ b/ci/release/build.sh @@ -26,31 +26,6 @@ Flags: EOF } -build() { - HW_BUILD_DIR="$BUILD_DIR/$OS/$ARCH/d2-$VERSION" - ARCHIVE="$BUILD_DIR/d2-$OS-$ARCH-$VERSION.tar.gz" - - if [ -e "$ARCHIVE" -a -z "${REBUILD-}" ]; then - log "skipping as already built at $ARCHIVE" - return 0 - fi - - sh_c mkdir -p "$HW_BUILD_DIR" - sh_c rsync --recursive --perms --delete \ - --human-readable --copy-links ./ci/release/template/ "$HW_BUILD_DIR/" - VERSION=$VERSION sh_c eval "'$HW_BUILD_DIR/README.md.sh'" \> "'$HW_BUILD_DIR/README.md'" - sh_c rm -f "$HW_BUILD_DIR/README.md.sh" - sh_c find "$HW_BUILD_DIR" -exec touch {} \; - - export GOOS=$(goos "$OS") - export GOARCH="$ARCH" - sh_c mkdir -p "$HW_BUILD_DIR/bin" - sh_c go build -ldflags "-X oss.terrastruct.com/d2/lib/version.Version=$VERSION" \ - -o "$HW_BUILD_DIR/bin/d2" ./cmd/d2 - - sh_c tar czf "$ARCHIVE" "$HW_BUILD_DIR" -} - main() { unset FLAG \ FLAGRAW \ @@ -108,4 +83,60 @@ main() { waitjobs } +build() { + HW_BUILD_DIR="$BUILD_DIR/$OS/$ARCH/d2-$VERSION" + ARCHIVE="$BUILD_DIR/d2-$OS-$ARCH-$VERSION.tar.gz" + + if [ -e "$ARCHIVE" -a -z "${REBUILD-}" ]; then + log "skipping as already built at $ARCHIVE" + return 0 + fi + + case $OS in + # macos) + # ;; + linux) + case $ARCH in + amd64) + RHOST=$TSTRUCT_LINUX_AMD64_BUILDER build_rhost + ;; + arm64) + RHOST=$TSTRUCT_LINUX_ARM64_BUILDER build_rhost + ;; + *) + COLOR=3 logp warn "no builder for OS=$OS, building locally..." + build_local + ;; + esac + ;; + *) + COLOR=3 logp warn "no builder for OS=$OS, building locally..." + # build_local + ;; + esac +} + +build_local() { + export DRYRUN \ + HW_BUILD_DIR \ + VERSION \ + OS \ + ARCH \ + ARCHIVE + sh_c ./ci/release/_build.sh +} + +build_rhost() { + sh_c ssh "$RHOST" mkdir -p src + sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/" + sh_c ssh -tttt "$RHOST" "DRYRUN=${DRYRUN-} \ +HW_BUILD_DIR=$HW_BUILD_DIR \ +VERSION=$VERSION \ +OS=$OS \ +ARCH=$ARCH \ +ARCHIVE=$ARCHIVE \ +./src/d2/ci/release/build_docker.sh" + sh_c rsync --archive --human-readable "$RHOST:src/d2/$ARCHIVE" "$ARCHIVE" +} + main "$@" diff --git a/ci/release/build_docker.sh b/ci/release/build_docker.sh new file mode 100755 index 000000000..f5505b7b7 --- /dev/null +++ b/ci/release/build_docker.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd -- "$(dirname "$0")/../.." +. ./ci/sub/lib.sh + +tag="$(sh_c docker build \ + -qf ./ci/release/builders/Dockerfile ./ci/release/builders )" +sh_c docker run -it --rm \ + -v "$HOME:$HOME" \ + -u "$(id -u):$(id -g)" \ + -w "$HOME" \ + -e DRYRUN="${DRYRUN-}" \ + -e HW_BUILD_DIR="$HW_BUILD_DIR" \ + -e VERSION="$VERSION" \ + -e OS="$OS" \ + -e ARCH="$ARCH" \ + -e ARCHIVE="$ARCHIVE" \ + -e TERM="$TERM" \ + "$tag" ./src/d2/ci/release/_build.sh diff --git a/ci/release/builders/Dockerfile b/ci/release/builders/Dockerfile new file mode 100644 index 000000000..59dd7f86c --- /dev/null +++ b/ci/release/builders/Dockerfile @@ -0,0 +1,11 @@ +FROM centos:7 + +ARG GOVERSION=1.19.3 + +RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" >/tmp/go.tar.gz +RUN tar -C /usr/local -xzf /tmp/go.tar.gz + +ENV PATH="/usr/local/go/bin:$PATH" + +RUN yum install -y rsync +RUN yum groupinstall -y 'Development Tools' diff --git a/ci/release/builders/create_aws.sh b/ci/release/builders/create_aws.sh new file mode 100755 index 000000000..4d71dc339 --- /dev/null +++ b/ci/release/builders/create_aws.sh @@ -0,0 +1,98 @@ +#!/bin/sh +set -eu +cd -- "$(dirname "$0")/../../.." +. ./ci/sub/lib.sh + +help() { + cat </dev/null \ + | jq -r .SecurityGroups[0].GroupId) + if [ -z "$SG_ID" ]; then + SG_ID=$(sh_c aws ec2 create-security-group \ + --group-name ssh \ + --description ssh \ + --vpc-id "$VPC_ID" | jq -r .GroupId) + fi + + header security-group-ingress + SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names ssh \ + | jq -r '.SecurityGroups[0].IpPermissions | length') + if [ "$SG_RULES_COUNT" -eq 0 ]; then + sh_c aws ec2 authorize-security-group-ingress \ + --group-id "$SG_ID" \ + --protocol tcp \ + --port 22 \ + --cidr 0.0.0.0/0 >/dev/null + fi + + header linux-amd64 + if ! aws ec2 describe-instances \ + "--query=Reservations[*].Instances[?State.Name!='terminated']" \ + | grep -q d2-builder-linux-amd64; then + sh_c aws ec2 run-instances \ + --image-id=ami-0d593311db5abb72b \ + --count=1 \ + --instance-type=t2.micro \ + --security-groups=ssh \ + "--key-name=$KEY_NAME" \ + --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' \ + 'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' >/dev/null + fi + + header linux-arm64 + if ! aws ec2 describe-instances \ + "--query=Reservations[*].Instances[?State.Name!='terminated']" \ + | grep -q d2-builder-linux-arm64; then + sh_c aws ec2 run-instances \ + --image-id=ami-0efabcf945ffd8831 \ + --count=1 \ + --instance-type=t4g.nano \ + --security-groups=ssh \ + "--key-name=$KEY_NAME" \ + --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' \ + 'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' >/dev/null + fi +} + +main "$@" diff --git a/ci/release/builders/init_aws.sh b/ci/release/builders/init_aws.sh index 0f85ddfec..34c54f4cd 100755 --- a/ci/release/builders/init_aws.sh +++ b/ci/release/builders/init_aws.sh @@ -7,7 +7,7 @@ help() { cat </dev/null \ - | jq -r .SecurityGroups[0].GroupId) - if [ -z "$SG_ID" ]; then - SG_ID=$(sh_c aws ec2 create-security-group \ - --group-name ssh \ - --description ssh \ - --vpc-id "$VPC_ID" | jq -r .GroupId) - fi - - header security-group-ingress - SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names ssh \ - | jq -r '.SecurityGroups[0].IpPermissions | length') - if [ "$SG_RULES_COUNT" -eq 0 ]; then - sh_c aws ec2 authorize-security-group-ingress \ - --group-id "$SG_ID" \ - --protocol tcp \ - --port 22 \ - --cidr 0.0.0.0/0 >/dev/null - fi - header linux-amd64 - if ! aws ec2 describe-instances \ - "--query=Reservations[*].Instances[?State.Name!='terminated']" \ - | grep -q d2-builder-linux-amd64; then - sh_c aws ec2 run-instances \ - --image-id=ami-0d593311db5abb72b \ - --count=1 \ - --instance-type=t2.micro \ - --security-groups=ssh \ - "--key-name=$KEY_NAME" \ - --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' \ - 'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' >/dev/null - fi - + # RHOST=$TSTRUCT_LINUX_AMD64_BUILDER init_rhost header linux-arm64 - if ! aws ec2 describe-instances \ - "--query=Reservations[*].Instances[?State.Name!='terminated']" \ - | grep -q d2-builder-linux-arm64; then - sh_c aws ec2 run-instances \ - --image-id=ami-0efabcf945ffd8831 \ - --count=1 \ - --instance-type=t4g.nano \ - --security-groups=ssh \ - "--key-name=$KEY_NAME" \ - --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' \ - 'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' >/dev/null - fi + RHOST=$TSTRUCT_LINUX_ARM64_BUILDER init_rhost +} + +init_rhost() { + sh_c ssh "$RHOST" 'sudo yum upgrade -y' + sh_c ssh "$RHOST" 'sudo yum install -y docker' + sh_c ssh "$RHOST" 'sudo systemctl start docker' + sh_c ssh "$RHOST" 'sudo systemctl enable docker' + sh_c ssh "$RHOST" 'sudo usermod -a -G docker ec2-user' + sh_c ssh "$RHOST" 'sudo reboot' || true } main "$@" diff --git a/ci/sub b/ci/sub index 4760501f6..7ede128e7 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit 4760501f65c5abace1790a501f34a7b1e3223b2f +Subproject commit 7ede128e71f0409b61dd9088aea22389f1b7b221