2022-11-13 03:10:45 +00:00
|
|
|
#!/bin/sh
|
|
|
|
|
set -eu
|
2022-12-15 15:14:06 +00:00
|
|
|
. "$(dirname "$0")/../../../ci/sub/lib.sh"
|
|
|
|
|
cd -- "$(dirname "$0")/../../.."
|
2022-11-13 03:10:45 +00:00
|
|
|
|
|
|
|
|
help() {
|
|
|
|
|
cat <<EOF
|
2022-12-15 15:14:06 +00:00
|
|
|
usage: $0 [--dry-run] [--skip-create] [--skip-init] [--copy-id=id.pub]
|
|
|
|
|
[--run=jobregex]
|
2022-11-13 03:10:45 +00:00
|
|
|
|
2022-11-13 04:39:14 +00:00
|
|
|
$0 creates and ensures the d2 builders in AWS.
|
2022-11-13 03:10:45 +00:00
|
|
|
EOF
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
main() {
|
2022-11-16 18:48:39 +00:00
|
|
|
while flag_parse "$@"; do
|
2022-11-13 03:10:45 +00:00
|
|
|
case "$FLAG" in
|
|
|
|
|
h|help)
|
|
|
|
|
help
|
|
|
|
|
return 0
|
|
|
|
|
;;
|
2022-12-14 08:47:07 +00:00
|
|
|
x)
|
|
|
|
|
flag_noarg && shift "$FLAGSHIFT"
|
|
|
|
|
set -x
|
|
|
|
|
export TRACE=1
|
|
|
|
|
;;
|
2022-11-14 06:59:09 +00:00
|
|
|
dry-run)
|
2022-11-14 05:58:55 +00:00
|
|
|
flag_noarg && shift "$FLAGSHIFT"
|
2022-12-14 08:47:07 +00:00
|
|
|
export DRY_RUN=1
|
2022-11-13 03:10:45 +00:00
|
|
|
;;
|
2022-12-14 08:47:07 +00:00
|
|
|
copy-id)
|
|
|
|
|
flag_nonemptyarg && shift "$FLAGSHIFT"
|
|
|
|
|
ID_PUB_PATH=$FLAGARG
|
|
|
|
|
;;
|
2022-12-15 15:14:06 +00:00
|
|
|
run)
|
|
|
|
|
flag_reqarg && shift "$FLAGSHIFT"
|
|
|
|
|
JOBFILTER="$FLAGARG"
|
|
|
|
|
;;
|
2022-11-13 03:10:45 +00:00
|
|
|
*)
|
|
|
|
|
flag_errusage "unrecognized flag $FLAGRAW"
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
done
|
2022-11-16 18:48:39 +00:00
|
|
|
shift "$FLAGSHIFT"
|
2022-11-13 03:10:45 +00:00
|
|
|
if [ $# -gt 0 ]; then
|
|
|
|
|
flag_errusage "no arguments are accepted"
|
|
|
|
|
fi
|
2022-12-14 08:47:07 +00:00
|
|
|
if [ -z "${ID_PUB_PATH-}" ]; then
|
|
|
|
|
flag_errusage "--copy-id is required"
|
|
|
|
|
fi
|
2022-11-13 03:10:45 +00:00
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
JOBNAME=create runjob_filter create_remote_hosts
|
|
|
|
|
JOBNAME=init && runjob_filter init_remote_hosts
|
2022-12-16 03:55:50 +00:00
|
|
|
|
|
|
|
|
FGCOLOR=2 header summary
|
|
|
|
|
echo "export CI_D2_LINUX_AMD64=$CI_D2_LINUX_AMD64"
|
|
|
|
|
echo "export CI_D2_LINUX_ARM64=$CI_D2_LINUX_ARM64"
|
|
|
|
|
echo "export CI_D2_MACOS_AMD64=$CI_D2_MACOS_AMD64"
|
|
|
|
|
echo "export CI_D2_MACOS_ARM64=$CI_D2_MACOS_ARM64"
|
|
|
|
|
echo "export CI_D2_WINDOWS_AMD64=$CI_D2_WINDOWS_AMD64"
|
2022-11-13 03:10:45 +00:00
|
|
|
}
|
|
|
|
|
|
2022-11-16 14:14:07 +00:00
|
|
|
create_remote_hosts() {
|
2022-12-14 08:47:07 +00:00
|
|
|
bigheader create_remote_hosts
|
|
|
|
|
|
2022-11-13 03:10:45 +00:00
|
|
|
KEY_NAME=$(aws ec2 describe-key-pairs | jq -r .KeyPairs[0].KeyName)
|
2022-12-15 15:14:06 +00:00
|
|
|
KEY_NAME_WINDOWS=windows
|
2022-11-13 03:10:45 +00:00
|
|
|
VPC_ID=$(aws ec2 describe-vpcs | jq -r .Vpcs[0].VpcId)
|
|
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
JOBNAME=$JOBNAME/security-groups runjob_filter create_security_groups
|
|
|
|
|
JOBNAME=$JOBNAME/linux/amd64 runjob_filter create_linux_amd64
|
|
|
|
|
JOBNAME=$JOBNAME/linux/arm64 runjob_filter create_linux_arm64
|
|
|
|
|
JOBNAME=$JOBNAME/macos/amd64 runjob_filter create_macos_amd64
|
|
|
|
|
JOBNAME=$JOBNAME/macos/arm64 runjob_filter create_macos_arm64
|
|
|
|
|
JOBNAME=$JOBNAME/windows/amd64 runjob_filter create_windows_amd64
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
create_security_groups() {
|
2022-11-13 03:10:45 +00:00
|
|
|
header security-group
|
|
|
|
|
SG_ID=$(aws ec2 describe-security-groups --group-names ssh 2>/dev/null \
|
|
|
|
|
| jq -r .SecurityGroups[0].GroupId)
|
|
|
|
|
if [ -z "$SG_ID" ]; then
|
|
|
|
|
SG_ID=$(sh_c aws ec2 create-security-group \
|
|
|
|
|
--group-name ssh \
|
|
|
|
|
--description ssh \
|
|
|
|
|
--vpc-id "$VPC_ID" | jq -r .GroupId)
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
header security-group-ingress
|
|
|
|
|
SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names ssh \
|
|
|
|
|
| jq -r '.SecurityGroups[0].IpPermissions | length')
|
|
|
|
|
if [ "$SG_RULES_COUNT" -eq 0 ]; then
|
|
|
|
|
sh_c aws ec2 authorize-security-group-ingress \
|
|
|
|
|
--group-id "$SG_ID" \
|
|
|
|
|
--protocol tcp \
|
|
|
|
|
--port 22 \
|
|
|
|
|
--cidr 0.0.0.0/0 >/dev/null
|
|
|
|
|
fi
|
|
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
header windows-security-group
|
|
|
|
|
SG_ID=$(aws ec2 describe-security-groups --group-names windows 2>/dev/null \
|
|
|
|
|
| jq -r .SecurityGroups[0].GroupId)
|
|
|
|
|
if [ -z "$SG_ID" ]; then
|
|
|
|
|
SG_ID=$(sh_c aws ec2 create-security-group \
|
|
|
|
|
--group-name windows \
|
|
|
|
|
--description windows \
|
|
|
|
|
--vpc-id "$VPC_ID" | jq -r .GroupId)
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
header windows-security-group-ingress
|
|
|
|
|
SG_RULES_COUNT=$(aws ec2 describe-security-groups --group-names windows \
|
|
|
|
|
| jq -r '.SecurityGroups[0].IpPermissions | length')
|
|
|
|
|
if [ "$SG_RULES_COUNT" -ne 2 ]; then
|
|
|
|
|
sh_c aws ec2 authorize-security-group-ingress \
|
|
|
|
|
--group-id "$SG_ID" \
|
|
|
|
|
--protocol tcp \
|
|
|
|
|
--port 22 \
|
|
|
|
|
--cidr 0.0.0.0/0 >/dev/null
|
|
|
|
|
sh_c aws ec2 authorize-security-group-ingress \
|
|
|
|
|
--group-id "$SG_ID" \
|
|
|
|
|
--protocol tcp \
|
|
|
|
|
--port 3389 \
|
|
|
|
|
--cidr 0.0.0.0/0 >/dev/null
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
create_linux_amd64() {
|
2022-11-13 03:10:45 +00:00
|
|
|
header linux-amd64
|
2022-12-15 15:14:06 +00:00
|
|
|
REMOTE_NAME=ci-d2-linux-amd64
|
2022-11-14 18:26:37 +00:00
|
|
|
state=$(aws ec2 describe-instances --filters \
|
2022-12-15 15:14:06 +00:00
|
|
|
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-linux-amd64' \
|
2022-11-14 18:26:37 +00:00
|
|
|
| jq -r '.Reservations[].Instances[].State.Name')
|
|
|
|
|
if [ -z "$state" ]; then
|
2022-11-13 03:10:45 +00:00
|
|
|
sh_c aws ec2 run-instances \
|
2022-12-14 08:47:07 +00:00
|
|
|
--image-id=ami-0ecc74eca1d66d8a6 \
|
2022-11-13 03:10:45 +00:00
|
|
|
--count=1 \
|
2022-12-15 15:14:06 +00:00
|
|
|
--instance-type=t3.small \
|
2022-11-13 03:10:45 +00:00
|
|
|
--security-groups=ssh \
|
|
|
|
|
"--key-name=$KEY_NAME" \
|
2022-12-14 08:47:07 +00:00
|
|
|
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
|
2022-12-15 15:14:06 +00:00
|
|
|
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=64,VolumeType=gp3}"' \
|
|
|
|
|
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-linux-amd64}]"' \
|
|
|
|
|
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-linux-amd64}]"' >/dev/null
|
2022-11-13 03:10:45 +00:00
|
|
|
fi
|
2022-12-15 15:14:06 +00:00
|
|
|
wait_remote_host_ip
|
|
|
|
|
log "CI_D2_LINUX_AMD64=ubuntu@$ip"
|
|
|
|
|
export CI_D2_LINUX_AMD64=ubuntu@$ip
|
|
|
|
|
}
|
2022-11-13 03:10:45 +00:00
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
create_linux_arm64() {
|
2022-11-13 03:10:45 +00:00
|
|
|
header linux-arm64
|
2022-12-15 15:14:06 +00:00
|
|
|
REMOTE_NAME=ci-d2-linux-arm64
|
2022-11-14 18:26:37 +00:00
|
|
|
state=$(aws ec2 describe-instances --filters \
|
2022-12-15 15:14:06 +00:00
|
|
|
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-linux-arm64' \
|
2022-11-14 18:26:37 +00:00
|
|
|
| jq -r '.Reservations[].Instances[].State.Name')
|
|
|
|
|
if [ -z "$state" ]; then
|
2022-11-13 03:10:45 +00:00
|
|
|
sh_c aws ec2 run-instances \
|
2022-12-14 08:47:07 +00:00
|
|
|
--image-id=ami-06e2dea2cdda3acda \
|
2022-11-13 03:10:45 +00:00
|
|
|
--count=1 \
|
2022-11-13 04:39:14 +00:00
|
|
|
--instance-type=t4g.small \
|
2022-11-13 03:10:45 +00:00
|
|
|
--security-groups=ssh \
|
|
|
|
|
"--key-name=$KEY_NAME" \
|
2022-12-14 08:47:07 +00:00
|
|
|
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
|
2022-12-15 15:14:06 +00:00
|
|
|
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=64,VolumeType=gp3}"' \
|
|
|
|
|
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-linux-arm64}]"' \
|
|
|
|
|
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-linux-arm64}]"' >/dev/null
|
2022-11-14 14:57:01 +00:00
|
|
|
fi
|
2022-12-15 15:14:06 +00:00
|
|
|
wait_remote_host_ip
|
|
|
|
|
log "CI_D2_LINUX_ARM64=ubuntu@$ip"
|
|
|
|
|
export CI_D2_LINUX_ARM64=ubuntu@$ip
|
|
|
|
|
}
|
2022-11-14 18:26:37 +00:00
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
create_macos_amd64() {
|
|
|
|
|
header macos-amd64-host
|
|
|
|
|
MACOS_AMD64_ID=$(aws ec2 describe-hosts --filter 'Name=state,Values=pending,available' 'Name=tag:Name,Values=ci-d2-macos-amd64' | jq -r '.Hosts[].HostId')
|
|
|
|
|
if [ -z "$MACOS_AMD64_ID" ]; then
|
|
|
|
|
MACOS_AMD64_ID=$(sh_c aws ec2 allocate-hosts --instance-type mac1.metal --quantity 1 --availability-zone us-west-2a \
|
|
|
|
|
--tag-specifications '"ResourceType=dedicated-host,Tags=[{Key=Name,Value=ci-d2-macos-amd64}]"' \
|
2022-11-14 18:26:37 +00:00
|
|
|
| jq -r .HostIds[0])
|
2022-11-14 14:57:01 +00:00
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
header macos-amd64
|
2022-12-15 15:14:06 +00:00
|
|
|
REMOTE_NAME=ci-d2-macos-amd64
|
2022-11-14 18:26:37 +00:00
|
|
|
state=$(aws ec2 describe-instances --filters \
|
2022-12-15 15:14:06 +00:00
|
|
|
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-macos-amd64' \
|
2022-11-14 18:26:37 +00:00
|
|
|
| jq -r '.Reservations[].Instances[].State.Name')
|
|
|
|
|
if [ -z "$state" ]; then
|
2022-11-14 14:57:01 +00:00
|
|
|
sh_c aws ec2 run-instances \
|
|
|
|
|
--image-id=ami-0dd2ded7568750663 \
|
|
|
|
|
--count=1 \
|
|
|
|
|
--instance-type=mac1.metal \
|
|
|
|
|
--security-groups=ssh \
|
|
|
|
|
"--key-name=$KEY_NAME" \
|
2022-12-14 08:47:07 +00:00
|
|
|
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
|
2022-12-15 15:14:06 +00:00
|
|
|
--placement "Tenancy=host,HostId=$MACOS_AMD64_ID" \
|
|
|
|
|
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=100,VolumeType=gp3}"' \
|
|
|
|
|
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-macos-amd64}]"' \
|
|
|
|
|
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-macos-amd64}]"' >/dev/null
|
|
|
|
|
fi
|
|
|
|
|
wait_remote_host_ip
|
|
|
|
|
log "CI_D2_MACOS_AMD64=ec2-user@$ip"
|
|
|
|
|
export CI_D2_MACOS_AMD64=ec2-user@$ip
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
create_macos_arm64() {
|
|
|
|
|
header macos-arm64-host
|
|
|
|
|
MACOS_ARM64_ID=$(aws ec2 describe-hosts --filter 'Name=state,Values=pending,available' 'Name=tag:Name,Values=ci-d2-macos-arm64' | jq -r '.Hosts[].HostId')
|
|
|
|
|
if [ -z "$MACOS_ARM64_ID" ]; then
|
|
|
|
|
MACOS_ARM64_ID=$(sh_c aws ec2 allocate-hosts --instance-type mac2.metal --quantity 1 --availability-zone us-west-2a \
|
2022-12-16 03:55:50 +00:00
|
|
|
--tag-specifications '"ResourceType=dedicated-host,Tags=[{Key=Name,Value=ci-d2-macos-arm64}]"' \
|
2022-12-15 15:14:06 +00:00
|
|
|
| jq -r .HostIds[0])
|
2022-11-14 14:57:01 +00:00
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
header macos-arm64
|
2022-12-15 15:14:06 +00:00
|
|
|
REMOTE_NAME=ci-d2-macos-arm64
|
2022-11-14 18:26:37 +00:00
|
|
|
state=$(aws ec2 describe-instances --filters \
|
2022-12-15 15:14:06 +00:00
|
|
|
'Name=instance-state-name,Values=pending,running,stopping,stopped' 'Name=tag:Name,Values=ci-d2-macos-arm64' \
|
2022-11-14 18:26:37 +00:00
|
|
|
| jq -r '.Reservations[].Instances[].State.Name')
|
|
|
|
|
if [ -z "$state" ]; then
|
2022-11-14 14:57:01 +00:00
|
|
|
sh_c aws ec2 run-instances \
|
|
|
|
|
--image-id=ami-0af0516ff2c43dbbe \
|
|
|
|
|
--count=1 \
|
|
|
|
|
--instance-type=mac2.metal \
|
|
|
|
|
--security-groups=ssh \
|
|
|
|
|
"--key-name=$KEY_NAME" \
|
2022-12-14 08:47:07 +00:00
|
|
|
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
|
2022-12-15 15:14:06 +00:00
|
|
|
--placement "Tenancy=host,HostId=$MACOS_ARM64_ID" \
|
|
|
|
|
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=100,VolumeType=gp3}"' \
|
|
|
|
|
--tag-specifications '"ResourceType=instance,Tags=[{Key=Name,Value=ci-d2-macos-arm64}]"' \
|
|
|
|
|
'"ResourceType=volume,Tags=[{Key=Name,Value=ci-d2-macos-arm64}]"' >/dev/null
|
|
|
|
|
fi
|
|
|
|
|
wait_remote_host_ip
|
|
|
|
|
log "CI_D2_MACOS_ARM64=ec2-user@$ip"
|
|
|
|
|
export CI_D2_MACOS_ARM64=ec2-user@$ip
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
create_windows_amd64() {
|
|
|
|
|
header windows-amd64
|
|
|
|
|
REMOTE_NAME=ci-d2-windows-amd64
|
|
|
|
|
state=$(aws ec2 describe-instances --filters \
|
|
|
|
|
'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
|
|
|
|
|
| jq -r '.Reservations[].Instances[].State.Name')
|
|
|
|
|
if [ -z "$state" ]; then
|
|
|
|
|
sh_c aws ec2 run-instances \
|
|
|
|
|
--image-id=ami-0c5300e833c2b32f3 \
|
|
|
|
|
--count=1 \
|
|
|
|
|
--instance-type=t3.medium \
|
|
|
|
|
--security-groups=windows \
|
|
|
|
|
"--key-name=$KEY_NAME_WINDOWS" \
|
|
|
|
|
--iam-instance-profile 'Name=AmazonSSMRoleForInstancesQuickSetup' \
|
|
|
|
|
--block-device-mappings '"DeviceName=/dev/sda1,Ebs={VolumeSize=64,VolumeType=gp3}"' \
|
|
|
|
|
--tag-specifications "'ResourceType=instance,Tags=[{Key=Name,Value=$REMOTE_NAME}]'" \
|
|
|
|
|
"'ResourceType=volume,Tags=[{Key=Name,Value=$REMOTE_NAME}]'" >/dev/null
|
2022-11-14 14:57:01 +00:00
|
|
|
fi
|
2022-12-15 15:14:06 +00:00
|
|
|
wait_remote_host_ip
|
|
|
|
|
log "CI_D2_WINDOWS_AMD64=Administrator@$ip"
|
|
|
|
|
export CI_D2_WINDOWS_AMD64=Administrator@$ip
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
wait_remote_host_ip() {
|
2022-11-14 14:57:01 +00:00
|
|
|
while true; do
|
2022-12-15 15:14:06 +00:00
|
|
|
ip=$(sh_c aws ec2 describe-instances \
|
|
|
|
|
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
|
|
|
|
|
| jq -r '.Reservations[].Instances[].PublicIpAddress')
|
|
|
|
|
if [ -n "$ip" ]; then
|
|
|
|
|
alloc_static_ip
|
|
|
|
|
ip=$(sh_c aws ec2 describe-instances \
|
|
|
|
|
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
|
|
|
|
|
| jq -r '.Reservations[].Instances[].PublicIpAddress')
|
2022-11-14 14:57:01 +00:00
|
|
|
break
|
|
|
|
|
fi
|
|
|
|
|
sleep 5
|
|
|
|
|
done
|
2022-11-13 03:10:45 +00:00
|
|
|
}
|
|
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
alloc_static_ip() {
|
|
|
|
|
allocation_id=$(aws ec2 describe-addresses --filters "Name=tag:Name,Values=$REMOTE_NAME" | jq -r '.Addresses[].AllocationId')
|
|
|
|
|
if [ -z "$allocation_id" ]; then
|
|
|
|
|
sh_c aws ec2 allocate-address --tag-specifications "'ResourceType=elastic-ip,Tags=[{Key=Name,Value=$REMOTE_NAME}]'"
|
|
|
|
|
allocation_id=$(aws ec2 describe-addresses --filters "Name=tag:Name,Values=$REMOTE_NAME" | jq -r '.Addresses[].AllocationId')
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
instance_id=$(aws ec2 describe-instances \
|
|
|
|
|
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
|
|
|
|
|
| jq -r '.Reservations[].Instances[].InstanceId')
|
|
|
|
|
aws ec2 associate-address --instance-id "$instance_id" --allocation-id "$allocation_id"
|
|
|
|
|
}
|
|
|
|
|
|
2022-11-16 14:14:07 +00:00
|
|
|
init_remote_hosts() {
|
2022-12-14 08:47:07 +00:00
|
|
|
bigheader init_remote_hosts
|
|
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
JOBNAME=$JOBNAME/linux/amd64 runjob_filter REMOTE_HOST=$CI_D2_LINUX_AMD64 REMOTE_NAME=ci-d2-linux-amd64 init_remote_linux
|
|
|
|
|
JOBNAME=$JOBNAME/linux/arm64 runjob_filter REMOTE_HOST=$CI_D2_LINUX_ARM64 REMOTE_NAME=ci-d2-linux-arm64 init_remote_linux
|
|
|
|
|
JOBNAME=$JOBNAME/macos/amd64 runjob_filter REMOTE_HOST=$CI_D2_MACOS_AMD64 REMOTE_NAME=ci-d2-macos-amd64 init_remote_macos
|
|
|
|
|
JOBNAME=$JOBNAME/macos/arm64 runjob_filter REMOTE_HOST=$CI_D2_MACOS_ARM64 REMOTE_NAME=ci-d2-macos-arm64 init_remote_macos
|
|
|
|
|
JOBNAME=$JOBNAME/windows/amd64 runjob_filter REMOTE_HOST=$CI_D2_WINDOWS_AMD64 REMOTE_NAME=ci-d2-windows-amd64 init_remote_windows
|
2022-11-14 14:57:01 +00:00
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
# Windows and AWS SSM both defeated me.
|
|
|
|
|
FGCOLOR=3 bigheader "WARNING: WINDOWS INITIALIZATION MUST BE COMPLETED MANUALLY OVER RDP AND POWERSHELL!"
|
2022-11-13 04:39:14 +00:00
|
|
|
}
|
|
|
|
|
|
2022-11-16 14:14:07 +00:00
|
|
|
init_remote_linux() {
|
2022-12-15 15:14:06 +00:00
|
|
|
header "$REMOTE_NAME"
|
2022-12-06 12:03:47 +00:00
|
|
|
wait_remote_host
|
2022-11-16 16:24:21 +00:00
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
sh_c ssh_copy_id -i="$ID_PUB_PATH" "$REMOTE_HOST"
|
2022-12-14 08:47:07 +00:00
|
|
|
|
2022-11-16 16:24:21 +00:00
|
|
|
sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
|
|
|
|
|
set -eux
|
|
|
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
|
|
|
|
|
|
|
|
sudo -E apt-get update -y
|
|
|
|
|
sudo -E apt-get dist-upgrade -y
|
2022-12-15 15:14:06 +00:00
|
|
|
sudo -E apt-get update -y
|
2022-11-16 16:24:21 +00:00
|
|
|
sudo -E apt-get install -y build-essential rsync
|
|
|
|
|
|
2022-12-14 08:47:07 +00:00
|
|
|
# Docker from https://docs.docker.com/engine/install/ubuntu/
|
2022-11-16 16:24:21 +00:00
|
|
|
sudo -E apt-get -y install \
|
|
|
|
|
ca-certificates \
|
|
|
|
|
curl \
|
|
|
|
|
gnupg \
|
|
|
|
|
lsb-release
|
|
|
|
|
sudo mkdir -p /etc/apt/keyrings
|
2022-12-14 08:47:07 +00:00
|
|
|
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --yes --dearmor -o /etc/apt/keyrings/docker.gpg
|
2022-11-16 16:24:21 +00:00
|
|
|
echo \
|
2022-12-14 08:47:07 +00:00
|
|
|
"deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
|
2022-11-16 16:24:21 +00:00
|
|
|
\$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
|
|
|
|
|
sudo -E apt-get update -y
|
|
|
|
|
sudo -E apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
|
|
|
|
sudo groupadd docker || true
|
|
|
|
|
sudo usermod -aG docker \$USER
|
2022-12-06 12:56:23 +00:00
|
|
|
|
|
|
|
|
mkdir -p \$HOME/.local/bin
|
|
|
|
|
mkdir -p \$HOME/.local/share/man
|
2022-11-16 16:24:21 +00:00
|
|
|
EOF
|
2022-12-06 12:03:47 +00:00
|
|
|
init_remote_env
|
2022-11-16 16:24:21 +00:00
|
|
|
|
2022-12-14 08:47:07 +00:00
|
|
|
sh_c ssh "$REMOTE_HOST" sh -s -- <<EOF
|
|
|
|
|
set -eux
|
|
|
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
|
|
|
sudo -E apt-get autoremove -y
|
|
|
|
|
EOF
|
2022-11-16 14:14:07 +00:00
|
|
|
sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
|
2022-11-13 04:39:14 +00:00
|
|
|
}
|
|
|
|
|
|
2022-11-16 14:14:07 +00:00
|
|
|
init_remote_macos() {
|
2022-12-15 15:14:06 +00:00
|
|
|
header "$REMOTE_NAME"
|
2022-12-06 12:03:47 +00:00
|
|
|
wait_remote_host
|
2022-12-06 07:22:38 +00:00
|
|
|
|
2022-11-16 16:24:21 +00:00
|
|
|
sh_c ssh "$REMOTE_HOST" '"/bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\""'
|
2022-12-06 07:22:38 +00:00
|
|
|
|
|
|
|
|
if sh_c ssh "$REMOTE_HOST" uname -m | grep -qF arm64; then
|
|
|
|
|
shellenv=$(sh_c ssh "$REMOTE_HOST" /opt/homebrew/bin/brew shellenv)
|
|
|
|
|
else
|
|
|
|
|
shellenv=$(sh_c ssh "$REMOTE_HOST" /usr/local/bin/brew shellenv)
|
|
|
|
|
fi
|
|
|
|
|
if ! echo "$shellenv" | sh_c ssh "$REMOTE_HOST" "IFS= read -r regex\; \"grep -qF \\\"\\\$regex\\\" ~/.zshrc\""; then
|
|
|
|
|
echo "$shellenv" | sh_c ssh "$REMOTE_HOST" "\"(echo && cat) >> ~/.zshrc\""
|
|
|
|
|
fi
|
2022-12-06 12:03:47 +00:00
|
|
|
if ! sh_c ssh "$REMOTE_HOST" "'grep -qF \\\$HOME/.local ~/.zshrc'"; then
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" "\"(echo && cat) >> ~/.zshrc\"" <<EOF
|
|
|
|
|
PATH=\$HOME/.local/bin:\$PATH
|
|
|
|
|
MANPATH=\$HOME/.local/share/man:\$MANPATH
|
|
|
|
|
EOF
|
|
|
|
|
fi
|
|
|
|
|
init_remote_env
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" brew update
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" brew upgrade
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" brew install go rsync
|
2022-12-14 08:47:07 +00:00
|
|
|
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" 'sudo reboot' || true
|
2022-12-06 12:03:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
init_remote_env() {
|
2022-12-06 07:22:38 +00:00
|
|
|
sh_c ssh "$REMOTE_HOST" '"rm -f ~/.ssh/environment"'
|
2022-12-06 12:03:47 +00:00
|
|
|
sh_c ssh "$REMOTE_HOST" '"echo PATH=\$(echo \"echo \\\$PATH\" | \"\$SHELL\" -ils) >\$HOME/.ssh/environment"'
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" '"echo MANPATH=\$(echo \"echo \\\$MANPATH\" | \"\$SHELL\" -ils) >>\$HOME/.ssh/environment"'
|
2022-12-06 07:22:38 +00:00
|
|
|
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" "sudo sed -i.bak '\"s/#PermitUserEnvironment no/PermitUserEnvironment yes/\"' /etc/ssh/sshd_config"
|
|
|
|
|
|
2022-12-06 12:03:47 +00:00
|
|
|
if sh_c ssh "$REMOTE_HOST" uname | grep -qF Darwin; then
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" "sudo launchctl stop com.openssh.sshd"
|
|
|
|
|
else
|
|
|
|
|
sh_c ssh "$REMOTE_HOST" "sudo systemctl restart sshd"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
wait_remote_host() {
|
|
|
|
|
while true; do
|
|
|
|
|
if sh_c ssh "$REMOTE_HOST" true; then
|
|
|
|
|
break
|
|
|
|
|
fi
|
|
|
|
|
sleep 5
|
|
|
|
|
done
|
2022-11-14 14:57:01 +00:00
|
|
|
}
|
|
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
wait_remote_host_windows() {
|
|
|
|
|
instance_id=$(aws ec2 describe-instances \
|
|
|
|
|
--filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" \
|
|
|
|
|
| jq -r '.Reservations[].Instances[].InstanceId')
|
|
|
|
|
|
|
|
|
|
while true; do
|
|
|
|
|
if sh_c aws ssm start-session --target "$instance_id" \
|
|
|
|
|
--document-name 'AWS-StartNonInteractiveCommand' \
|
|
|
|
|
--parameters "'{\"command\": [\"echo true; exit\"]}'"; then
|
|
|
|
|
break
|
|
|
|
|
fi
|
|
|
|
|
sleep 5
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
init_remote_windows() {
|
|
|
|
|
header "$REMOTE_NAME"
|
|
|
|
|
wait_remote_host_windows
|
|
|
|
|
|
|
|
|
|
init_ps1=$(cat <<EOF
|
|
|
|
|
\$ProgressPreference = 'SilentlyContinue'
|
|
|
|
|
|
2022-12-16 09:53:06 +00:00
|
|
|
# Bootstrap PowerShell v7
|
|
|
|
|
if ((\$PSVersionTable.PSVersion).Major -eq 5) {
|
|
|
|
|
Invoke-WebRequest -Uri https://www.nuget.org/api/v2/package/Microsoft.UI.Xaml/2.7.3 -OutFile .\microsoft.ui.xaml.2.7.3.zip
|
|
|
|
|
Expand-Archive -Force .\microsoft.ui.xaml.2.7.3.zip
|
|
|
|
|
Add-AppxPackage .\microsoft.ui.xaml.2.7.3\tools\AppX\x64\Release\Microsoft.UI.Xaml.2.7.appx
|
|
|
|
|
|
|
|
|
|
Invoke-WebRequest -Uri https://github.com/microsoft/winget-cli/releases/download/v1.3.2691/Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle -OutFile .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle
|
|
|
|
|
Invoke-WebRequest -Uri https://github.com/microsoft/winget-cli/releases/download/v1.3.2691/7bcb1a0ab33340daa57fa5b81faec616_License1.xml -OutFile .\7bcb1a0ab33340daa57fa5b81faec616_License1.xml
|
|
|
|
|
Invoke-WebRequest -Uri https://aka.ms/Microsoft.VCLibs.x64.14.00.Desktop.appx -OutFile Microsoft.VCLibs.x64.14.00.Desktop.appx
|
|
|
|
|
Add-AppxProvisionedPackage -online -PackagePath .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle -LicensePath .\7bcb1a0ab33340daa57fa5b81faec616_License1.xml -DependencyPackagePath Microsoft.VCLibs.x64.14.00.Desktop.appx
|
|
|
|
|
Add-AppxPackage .\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle
|
|
|
|
|
|
|
|
|
|
winget install --silent --accept-package-agreements --accept-source-agreements Microsoft.DotNet.SDK.7
|
|
|
|
|
# Refresh env.
|
|
|
|
|
\$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
|
|
|
|
|
dotnet tool install --global PowerShell --version 7.3.1
|
|
|
|
|
pwsh -c 'Enable-ExperimentalFeature PSNativeCommandErrorActionPreference'
|
|
|
|
|
pwsh .\Desktop\init.ps1
|
|
|
|
|
Exit
|
|
|
|
|
}
|
2022-12-15 15:14:06 +00:00
|
|
|
|
2022-12-16 09:53:06 +00:00
|
|
|
Set-StrictMode -Version Latest
|
|
|
|
|
\$ErrorActionPreference = "Stop"
|
|
|
|
|
\$PSNativeCommandUseErrorActionPreference = \$true
|
2022-12-15 15:14:06 +00:00
|
|
|
|
2022-12-16 09:53:06 +00:00
|
|
|
if (-Not (Get-Command wix -errorAction SilentlyContinue)) {
|
|
|
|
|
dotnet tool install --global wix --version 4.0.0-preview.1
|
|
|
|
|
}
|
2022-12-15 15:14:06 +00:00
|
|
|
|
|
|
|
|
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
|
|
|
|
|
Start-Service sshd
|
|
|
|
|
Set-Service -Name sshd -StartupType 'Automatic'
|
|
|
|
|
|
2022-12-16 09:53:06 +00:00
|
|
|
New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" -Name DefaultShell -Value "C:\Users\Administrator\.dotnet\tools\pwsh.exe" -PropertyType String -Force
|
|
|
|
|
|
2022-12-15 15:14:06 +00:00
|
|
|
ConvertFrom-Json -InputObject @'
|
|
|
|
|
$(perl -pe 's#\n#\r\n#' "$ID_PUB_PATH" | jq -Rs .)
|
|
|
|
|
'@ | Out-File -Encoding utf8 "\$env:ProgramData\ssh\administrators_authorized_keys"
|
|
|
|
|
# utf8BOM -> utf8: https://stackoverflow.com/a/34969243/4283659
|
|
|
|
|
\$null = New-Item -Force "\$env:ProgramData\ssh\administrators_authorized_keys" -Value (Get-Content -Path "\$env:ProgramData\ssh\administrators_authorized_keys" | Out-String)
|
|
|
|
|
get-acl "\$env:ProgramData\ssh\ssh_host_rsa_key" | set-acl "\$env:ProgramData\ssh\administrators_authorized_keys"
|
2022-12-16 03:55:50 +00:00
|
|
|
|
2022-12-16 09:53:06 +00:00
|
|
|
if (-Not (Test-Path -Path C:\msys64)) {
|
|
|
|
|
Invoke-WebRequest -Uri "https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-x86_64-20221028.exe" -OutFile "./msys2-x86_64.exe"
|
|
|
|
|
./msys2-x86_64.exe install --default-answer --confirm-command --root C:\msys64
|
|
|
|
|
}
|
|
|
|
|
C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c 'pacman -Sy --noconfirm base-devel vim rsync'
|
|
|
|
|
C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c 'curl -fsSL https://d2lang.com/install.sh | sh -s -- --tala'
|
|
|
|
|
C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c 'd2 --version'
|
2022-12-16 07:56:40 +00:00
|
|
|
|
2022-12-16 09:53:06 +00:00
|
|
|
\$oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path
|
|
|
|
|
\$newpath = "\$oldpath;C:\msys64\usr\bin"
|
|
|
|
|
Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value \$newPath
|
2022-12-15 15:14:06 +00:00
|
|
|
EOF
|
2022-12-16 08:24:59 +00:00
|
|
|
|
2022-12-16 08:59:50 +00:00
|
|
|
# To run a POSIX script:
|
|
|
|
|
# ssh "$CI_D2_WINDOWS_AMD64" sh -s -- <<EOF
|
|
|
|
|
# which wix
|
|
|
|
|
# EOF
|
|
|
|
|
# To run a command in a pure MSYS2 shell:
|
2022-12-16 08:24:59 +00:00
|
|
|
# ssh "$CI_D2_WINDOWS_AMD64" 'C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c "echo hi"'
|
2022-12-16 08:59:50 +00:00
|
|
|
# To run a pure MSYS2 shell:
|
|
|
|
|
# ssh -t "$CI_D2_WINDOWS_AMD64" 'C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64'
|
2022-12-16 08:24:59 +00:00
|
|
|
|
2022-12-16 08:40:51 +00:00
|
|
|
# In case MSYS2 improves in the future and allows for noninteractive commands the
|
|
|
|
|
# following will set the OpenSSH shell to MSYS2 instead of PowerShell.
|
|
|
|
|
#
|
|
|
|
|
# Right now, setting MSYS2 to the DefaultShell like this will make it start bash in
|
|
|
|
|
# interactive mode always. Even for ssh "$CI_D2_WINDOWS_AMD64" echo hi. And so you'll end
|
|
|
|
|
# up with a blank prompt on which to input commands instead of having it execute the
|
|
|
|
|
# command you passed in via ssh.
|
|
|
|
|
#
|
|
|
|
|
# PowerShell as the default is better anyway as it gives us access to both the UNIX
|
|
|
|
|
# userspace and Windows tools like wix/dotnet/winget.
|
|
|
|
|
#
|
|
|
|
|
# To set:
|
2022-12-16 08:24:59 +00:00
|
|
|
# <<EOF
|
|
|
|
|
# echo '@C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64' | Out-File C:\msys64\sshd_default_shell.cmd
|
|
|
|
|
# # utf8BOM -> utf8: https://stackoverflow.com/a/34969243/4283659
|
|
|
|
|
# \$null = New-Item -Force C:\msys64\sshd_default_shell.cmd -Value (Get-Content -Path C:\msys64\sshd_default_shell.cmd | Out-String)
|
|
|
|
|
# Set-ItemProperty -Path HKLM:\SOFTWARE\OpenSSH -Name DefaultShell -Value C:\msys64\sshd_default_shell.cmd
|
|
|
|
|
# EOF
|
2022-12-16 08:40:51 +00:00
|
|
|
#
|
2022-12-16 08:24:59 +00:00
|
|
|
# To undo:
|
|
|
|
|
# <<EOF
|
|
|
|
|
# Remove-ItemProperty -Path HKLM:\SOFTWARE\OpenSSH -Name DefaultShell
|
|
|
|
|
# rm C:\msys64\sshd_default_shell.cmd
|
|
|
|
|
# EOF
|
2022-12-15 15:14:06 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
gen_init_ps1=$(cat <<EOF
|
|
|
|
|
ConvertFrom-Json -InputObject @'
|
|
|
|
|
$(printf %s "$init_ps1" | perl -pe 'chomp if eof' | perl -pe 's#\n#\r\n#' | jq -Rs .)
|
|
|
|
|
'@ | Out-File -Encoding utf8 C:\Users\Administrator\Desktop\init.ps1; C:\Users\Administrator\Desktop\init.ps1
|
|
|
|
|
EOF
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Windows and AWS SSM both defeated me.
|
|
|
|
|
FGCOLOR=3 bigheader "WARNING: WINDOWS INITIALIZATION MUST BE COMPLETED MANUALLY OVER RDP AND POWERSHELL!"
|
|
|
|
|
|
|
|
|
|
warn '1. Obtain Windows RDP password with:'
|
|
|
|
|
echo " aws ec2 get-password-data --instance-id \$(aws ec2 describe-instances --filters 'Name=instance-state-name,Values=pending,running,stopping,stopped' "Name=tag:Name,Values=$REMOTE_NAME" | jq -r '.Reservations[].Instances[].InstanceId') --priv-launch-key windows.pem | jq -r .PasswordData" >&2
|
|
|
|
|
warn "2. RDP into $REMOTE_HOST and open PowerShell."
|
|
|
|
|
warn '3. Generate and execute C:\Users\Administrator\Desktop\init.ps1 with:'
|
|
|
|
|
printf '%s\n' "$gen_init_ps1" >&2
|
|
|
|
|
}
|
|
|
|
|
|
2022-11-13 03:10:45 +00:00
|
|
|
main "$@"
|