2022-11-12 07:10:45pm

ci/release/_build.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+cd -- "$(dirname "$0")/../.."
+. ./ci/sub/lib.sh
+
+sh_c mkdir -p "$HW_BUILD_DIR"
+sh_c rsync --recursive --perms --delete \
+  --human-readable --copy-links ./ci/release/template/ "$HW_BUILD_DIR/"
+VERSION=$VERSION sh_c eval "'$HW_BUILD_DIR/README.md.sh'" \> "'$HW_BUILD_DIR/README.md'"
+sh_c rm -f "$HW_BUILD_DIR/README.md.sh"
+sh_c find "$HW_BUILD_DIR" -exec touch {} \;
+
+export GOOS=$(goos "$OS")
+export GOARCH="$ARCH"
+sh_c mkdir -p "$HW_BUILD_DIR/bin"
+sh_c go build -ldflags "-X oss.terrastruct.com/d2/lib/version.Version=$VERSION" \
+  -o "$HW_BUILD_DIR/bin/d2" ./cmd/d2
+
+sh_c tar czf "$ARCHIVE" "$HW_BUILD_DIR"

ci/release/build.sh

@@ -26,31 +26,6 @@ Flags:
 EOF
 }
 
-build() {
-  HW_BUILD_DIR="$BUILD_DIR/$OS/$ARCH/d2-$VERSION"
-  ARCHIVE="$BUILD_DIR/d2-$OS-$ARCH-$VERSION.tar.gz"
-
-  if [ -e "$ARCHIVE" -a -z "${REBUILD-}" ]; then
-    log "skipping as already built at $ARCHIVE"
-    return 0
-  fi
-
-  sh_c mkdir -p "$HW_BUILD_DIR"
-  sh_c rsync --recursive --perms --delete \
-    --human-readable --copy-links ./ci/release/template/ "$HW_BUILD_DIR/"
-  VERSION=$VERSION sh_c eval "'$HW_BUILD_DIR/README.md.sh'" \> "'$HW_BUILD_DIR/README.md'"
-  sh_c rm -f "$HW_BUILD_DIR/README.md.sh"
-  sh_c find "$HW_BUILD_DIR" -exec touch {} \;
-
-  export GOOS=$(goos "$OS")
-  export GOARCH="$ARCH"
-  sh_c mkdir -p "$HW_BUILD_DIR/bin"
-  sh_c go build -ldflags "-X oss.terrastruct.com/d2/lib/version.Version=$VERSION" \
-    -o "$HW_BUILD_DIR/bin/d2" ./cmd/d2
-
-  sh_c tar czf "$ARCHIVE" "$HW_BUILD_DIR"
-}
-
 main() {
   unset FLAG \
     FLAGRAW \
@@ -108,4 +83,60 @@ main() {
   waitjobs
 }
 
+build() {
+  HW_BUILD_DIR="$BUILD_DIR/$OS/$ARCH/d2-$VERSION"
+  ARCHIVE="$BUILD_DIR/d2-$OS-$ARCH-$VERSION.tar.gz"
+
+  if [ -e "$ARCHIVE" -a -z "${REBUILD-}" ]; then
+    log "skipping as already built at $ARCHIVE"
+    return 0
+  fi
+
+  case $OS in
+    # macos)
+    #   ;;
+    linux)
+      case $ARCH in
+        amd64)
+          RHOST=$TSTRUCT_LINUX_AMD64_BUILDER build_rhost
+          ;;
+        arm64)
+          RHOST=$TSTRUCT_LINUX_ARM64_BUILDER build_rhost
+          ;;
+        *)
+          COLOR=3 logp warn "no builder for OS=$OS, building locally..."
+          build_local
+          ;;
+      esac
+      ;;
+    *)
+      COLOR=3 logp warn "no builder for OS=$OS, building locally..."
+      # build_local
+      ;;
+  esac
+}
+
+build_local() {
+  export DRYRUN \
+    HW_BUILD_DIR \
+    VERSION \
+    OS \
+    ARCH \
+    ARCHIVE
+  sh_c ./ci/release/_build.sh
+}
+
+build_rhost() {
+  sh_c ssh "$RHOST" mkdir -p src
+  sh_c rsync --archive --human-readable --delete ./ "$RHOST:src/d2/"
+  sh_c ssh -tttt "$RHOST" "DRYRUN=${DRYRUN-} \
+HW_BUILD_DIR=$HW_BUILD_DIR \
+VERSION=$VERSION \
+OS=$OS \
+ARCH=$ARCH \
+ARCHIVE=$ARCHIVE \
+./src/d2/ci/release/build_docker.sh"
+  sh_c rsync --archive --human-readable "$RHOST:src/d2/$ARCHIVE" "$ARCHIVE"
+}
+
 main "$@"

ci/release/build_docker.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+cd -- "$(dirname "$0")/../.."
+. ./ci/sub/lib.sh
+
+tag="$(sh_c docker build \
+  -qf ./ci/release/builders/Dockerfile ./ci/release/builders )"
+sh_c docker run -it --rm \
+  -v "$HOME:$HOME" \
+  -u "$(id -u):$(id -g)" \
+  -w "$HOME" \
+  -e DRYRUN="${DRYRUN-}" \
+  -e HW_BUILD_DIR="$HW_BUILD_DIR" \
+  -e VERSION="$VERSION" \
+  -e OS="$OS" \
+  -e ARCH="$ARCH" \
+  -e ARCHIVE="$ARCHIVE" \
+  -e TERM="$TERM" \
+  "$tag" ./src/d2/ci/release/_build.sh

ci/release/builders/Dockerfile

@@ -0,0 +1,11 @@
+FROM centos:7
+
+ARG GOVERSION=1.19.3
+
+RUN curl -fsSL "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" >/tmp/go.tar.gz
+RUN tar -C /usr/local -xzf /tmp/go.tar.gz
+
+ENV PATH="/usr/local/go/bin:$PATH"
+
+RUN yum install -y rsync
+RUN yum groupinstall -y 'Development Tools'

ci/release/builders/create_aws.sh

@@ -0,0 +1,98 @@
+#!/bin/sh
+set -eu
+cd -- "$(dirname "$0")/../../.."
+. ./ci/sub/lib.sh
+
+help() {
+  cat <<EOF
+usage: $0 [--dryrun]
+
+$0 creates the d2 builders in a AWS account.
+EOF
+}
+
+main() {
+  unset DRYRUN 
+  while :; do
+    flag_parse "$@"
+    case "$FLAG" in
+      h|help)
+        help
+        return 0
+        ;;
+      dryrun)
+        flag_noarg
+        DRYRUN=1
+        shift "$FLAGSHIFT"
+        ;;
+      '')
+        shift "$FLAGSHIFT"
+        break
+        ;;
+      *)
+        flag_errusage "unrecognized flag $FLAGRAW"
+        ;;
+    esac
+  done
+  if [ $# -gt 0 ]; then
+    flag_errusage "no arguments are accepted"
+  fi
+
+  create_aws
+}
+
+create_aws() {
+  KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName)
+  VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId)
+
+  header security-group
+  SG_ID=$(aws ec2 describe-security-groups --group-names ssh 2>/dev/null \
+    | jq -r .SecurityGroups[0].GroupId)
+  if [ -z "$SG_ID" ]; then
+    SG_ID=$(sh_c aws ec2 create-security-group \
+      --group-name ssh \
+      --description ssh \
+      --vpc-id "$VPC_ID" | jq -r .GroupId)
+  fi
+
+  header security-group-ingress
+  SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names ssh \
+    | jq -r '.SecurityGroups[0].IpPermissions | length')
+  if [ "$SG_RULES_COUNT" -eq 0 ]; then
+    sh_c aws ec2 authorize-security-group-ingress \
+      --group-id "$SG_ID" \
+      --protocol tcp \
+      --port 22 \
+      --cidr 0.0.0.0/0 >/dev/null
+  fi
+
+  header linux-amd64
+  if ! aws ec2 describe-instances \
+    "--query=Reservations[*].Instances[?State.Name!='terminated']" \
+    | grep -q d2-builder-linux-amd64; then
+    sh_c aws ec2 run-instances \
+      --image-id=ami-0d593311db5abb72b \
+      --count=1 \
+      --instance-type=t2.micro \
+      --security-groups=ssh \
+      "--key-name=$KEY_NAME" \
+      --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' \
+        'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' >/dev/null
+  fi
+
+  header linux-arm64
+  if ! aws ec2 describe-instances \
+    "--query=Reservations[*].Instances[?State.Name!='terminated']" \
+    | grep -q d2-builder-linux-arm64; then
+    sh_c aws ec2 run-instances \
+      --image-id=ami-0efabcf945ffd8831 \
+      --count=1 \
+      --instance-type=t4g.nano \
+      --security-groups=ssh \
+      "--key-name=$KEY_NAME" \
+      --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' \
+        'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' >/dev/null
+  fi
+}
+
+main "$@"

ci/release/builders/init_aws.sh

@@ -7,7 +7,7 @@ help() {
   cat <<EOF
 usage: $0 [--dryrun]
 
-$0 sets up an AWS account with the d2 builders.
+$0 inits the D2 builders by installing docker.
 EOF
 }
 
@@ -42,57 +42,19 @@ main() {
 }
 
 init_aws() {
-  KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName)
-  VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId)
-
-  header security-group
-  SG_ID=$(aws ec2 describe-security-groups --group-names ssh 2>/dev/null \
-    | jq -r .SecurityGroups[0].GroupId)
-  if [ -z "$SG_ID" ]; then
-    SG_ID=$(sh_c aws ec2 create-security-group \
-      --group-name ssh \
-      --description ssh \
-      --vpc-id "$VPC_ID" | jq -r .GroupId)
-  fi
-
-  header security-group-ingress
-  SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names ssh \
-    | jq -r '.SecurityGroups[0].IpPermissions | length')
-  if [ "$SG_RULES_COUNT" -eq 0 ]; then
-    sh_c aws ec2 authorize-security-group-ingress \
-      --group-id "$SG_ID" \
-      --protocol tcp \
-      --port 22 \
-      --cidr 0.0.0.0/0 >/dev/null
-  fi
-
   header linux-amd64
-  if ! aws ec2 describe-instances \
+  # RHOST=$TSTRUCT_LINUX_AMD64_BUILDER init_rhost
-    "--query=Reservations[*].Instances[?State.Name!='terminated']" \
-    | grep -q d2-builder-linux-amd64; then
-    sh_c aws ec2 run-instances \
-      --image-id=ami-0d593311db5abb72b \
-      --count=1 \
-      --instance-type=t2.micro \
-      --security-groups=ssh \
-      "--key-name=$KEY_NAME" \
-      --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' \
-        'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-amd64}]' >/dev/null
-  fi
-
   header linux-arm64
-  if ! aws ec2 describe-instances \
+  RHOST=$TSTRUCT_LINUX_ARM64_BUILDER init_rhost
-    "--query=Reservations[*].Instances[?State.Name!='terminated']" \
+}
-    | grep -q d2-builder-linux-arm64; then
+
-    sh_c aws ec2 run-instances \
+init_rhost() {
-      --image-id=ami-0efabcf945ffd8831 \
+  sh_c ssh "$RHOST" 'sudo yum upgrade -y'
-      --count=1 \
+  sh_c ssh "$RHOST" 'sudo yum install -y docker'
-      --instance-type=t4g.nano \
+  sh_c ssh "$RHOST" 'sudo systemctl start docker'
-      --security-groups=ssh \
+  sh_c ssh "$RHOST" 'sudo systemctl enable docker'
-      "--key-name=$KEY_NAME" \
+  sh_c ssh "$RHOST" 'sudo usermod -a -G docker ec2-user'
-      --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' \
+  sh_c ssh "$RHOST" 'sudo reboot' || true
-        'ResourceType=volume,Tags=[{Key=Name,Value=d2-builder-linux-arm64}]' >/dev/null
-  fi
 }
 
 main "$@"

ci/sub

@@ -1 +1 @@
-Subproject commit 4760501f65c5abace1790a501f34a7b1e3223b2f
+Subproject commit 7ede128e71f0409b61dd9088aea22389f1b7b221