From 84141efe37b7ba4724f06419474b02119bc0e18a Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 16 Nov 2022 10:48:39 -0800 Subject: [PATCH] ci: Many updates --- .github/workflows/ci.yml | 12 +- .github/workflows/daily.yml | 2 +- ci/release/_install.sh | 10 +- ci/release/build.sh | 9 +- ci/release/builders/aws_copy_keys.sh | 8 +- ci/release/builders/aws_ensure.sh | 10 +- ci/release/builders/ssh.sh | 8 +- ci/release/gen_template_lib.sh | 29 +++++ ci/release/template/scripts/lib.sh | 171 +++++++++++++++++++++++---- ci/sub | 2 +- go.sum | 2 + install.sh | 69 +++++------ 12 files changed, 232 insertions(+), 100 deletions(-) create mode 100755 ci/release/gen_template_lib.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73f3d930c..1727e3638 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - run: TERM=xterm-256color ./make.sh assert-linear + - run: COLOR=1 ./make.sh assert-linear env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} @@ -25,7 +25,7 @@ jobs: with: go-version-file: ./go.mod cache: true - - run: TERM=xterm-256color ./make.sh fmt + - run: COLOR=1 ./make.sh fmt env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} @@ -39,7 +39,7 @@ jobs: with: go-version-file: ./go.mod cache: true - - run: TERM=xterm-256color ./make.sh lint + - run: COLOR=1 ./make.sh lint env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} @@ -53,7 +53,7 @@ jobs: with: go-version-file: ./go.mod cache: true - - run: TERM=xterm-256color ./make.sh build + - run: COLOR=1 ./make.sh build env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} @@ -67,7 +67,7 @@ jobs: with: go-version-file: ./go.mod cache: true - - run: TERM=xterm-256color ./make.sh test + - run: COLOR=1 ./make.sh test env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} @@ -86,7 +86,7 @@ jobs: with: go-version-file: ./go.mod cache: true - - run: TERM=xterm-256color ./make.sh race + - run: COLOR=1 ./make.sh race env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index ed2e89be5..6334923c1 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -20,7 +20,7 @@ jobs: with: go-version-file: ./go.mod cache: true - - run: CI_ALL=1 TERM=xterm-256color ./make.sh + - run: CI_ALL=1 COLOR=1 ./make.sh env: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN }} DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} diff --git a/ci/release/_install.sh b/ci/release/_install.sh index 2917323ad..833f965f8 100755 --- a/ci/release/_install.sh +++ b/ci/release/_install.sh @@ -86,8 +86,7 @@ EOF main() { METHOD=standalone - while :; do - flag_parse "$@" + while flag_parse "$@"; do case "$FLAG" in h|help) help @@ -129,15 +128,12 @@ main() { flag_noarg && shift "$FLAGSHIFT" UNINSTALL=1 ;; - '') - shift "$FLAGSHIFT" - break - ;; *) flag_errusage "unrecognized flag $FLAGRAW" ;; esac done + shift "$FLAGSHIFT" if [ $# -gt 0 ]; then flag_errusage "no arguments are accepted" @@ -176,7 +172,7 @@ install() { TALA_VERSION="$( install_tala && echo "$VERSION" )" fi - COLOR=2 header success + FGCOLOR=2 header success log "d2-$VERSION-$OS-$ARCH has been successfully installed into $PREFIX" if [ -n "${TALA-}" ]; then log "tala-$TALA_VERSION-$OS-$ARCH has been successfully installed into $PREFIX" diff --git a/ci/release/build.sh b/ci/release/build.sh index 135160f82..04cdc7217 100755 --- a/ci/release/build.sh +++ b/ci/release/build.sh @@ -42,8 +42,7 @@ EOF } main() { - while :; do - flag_parse "$@" + while flag_parse "$@"; do case "$FLAG" in h|help) help @@ -78,16 +77,12 @@ main() { flag_noarg && shift "$FLAGSHIFT" LOCKFILE_FORCE=1 ;; - '') - shift "$FLAGSHIFT" - break - ;; *) flag_errusage "unrecognized flag $FLAGRAW" ;; esac done - + shift "$FLAGSHIFT" if [ $# -gt 0 ]; then flag_errusage "no arguments are accepted" fi diff --git a/ci/release/builders/aws_copy_keys.sh b/ci/release/builders/aws_copy_keys.sh index df5c46408..cc0d548ef 100755 --- a/ci/release/builders/aws_copy_keys.sh +++ b/ci/release/builders/aws_copy_keys.sh @@ -12,8 +12,7 @@ EOF } main() { - while :; do - flag_parse "$@" + while flag_parse "$@"; do case "$FLAG" in h|help) help @@ -27,15 +26,12 @@ main() { flag_nonemptyarg && shift "$FLAGSHIFT" KEY_FILE=$FLAGARG ;; - '') - shift "$FLAGSHIFT" - break - ;; *) flag_errusage "unrecognized flag $FLAGRAW" ;; esac done + shift "$FLAGSHIFT" if [ -z "${KEY_FILE-}" ]; then echoerr "-i is required" exit 1 diff --git a/ci/release/builders/aws_ensure.sh b/ci/release/builders/aws_ensure.sh index 3a875d832..bfb630ed0 100755 --- a/ci/release/builders/aws_ensure.sh +++ b/ci/release/builders/aws_ensure.sh @@ -12,8 +12,7 @@ EOF } main() { - while :; do - flag_parse "$@" + while flag_parse "$@"; do case "$FLAG" in h|help) help @@ -27,15 +26,12 @@ main() { flag_noarg && shift "$FLAGSHIFT" SKIP_CREATE=1 ;; - '') - shift "$FLAGSHIFT" - break - ;; *) flag_errusage "unrecognized flag $FLAGRAW" ;; esac done + shift "$FLAGSHIFT" if [ $# -gt 0 ]; then flag_errusage "no arguments are accepted" fi @@ -204,7 +200,7 @@ init_remote_hosts() { header macos-arm64 REMOTE_HOST=$TSTRUCT_MACOS_ARM64_BUILDER init_remote_macos - COLOR=2 header summary + FGCOLOR=2 header summary log "export TSTRUCT_LINUX_AMD64_BUILDER=$TSTRUCT_LINUX_AMD64_BUILDER" log "export TSTRUCT_LINUX_ARM64_BUILDER=$TSTRUCT_LINUX_ARM64_BUILDER" log "export TSTRUCT_MACOS_AMD64_BUILDER=$TSTRUCT_MACOS_AMD64_BUILDER" diff --git a/ci/release/builders/ssh.sh b/ci/release/builders/ssh.sh index 0325a99dd..3d0e7e995 100755 --- a/ci/release/builders/ssh.sh +++ b/ci/release/builders/ssh.sh @@ -12,8 +12,7 @@ EOF } main() { - while :; do - flag_parse "$@" + while flag_parse "$@"; do case "$FLAG" in h|help) help @@ -27,15 +26,12 @@ main() { flag_reqarg && shift "$FLAGSHIFT" JOBFILTER="$FLAGARG" ;; - '') - shift "$FLAGSHIFT" - break - ;; *) flag_errusage "unrecognized flag $FLAGRAW" ;; esac done + shift "$FLAGSHIFT" REMOTE_HOST=$TSTRUCT_LINUX_AMD64_BUILDER; runjob linux-amd64 ssh "$REMOTE_HOST" "$@" REMOTE_HOST=$TSTRUCT_LINUX_ARM64_BUILDER; runjob linux-arm64 ssh "$REMOTE_HOST" "$@" diff --git a/ci/release/gen_template_lib.sh b/ci/release/gen_template_lib.sh new file mode 100755 index 000000000..1f523767a --- /dev/null +++ b/ci/release/gen_template_lib.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu +cd -- "$(dirname "$0")/../.." +. ./ci/sub/lib.sh + +sh_c chmod +w ./ci/release/template/scripts/lib.sh +sh_c cat >./ci/release/template/scripts/lib.sh <\>./ci/release/template/scripts/lib.sh +sh_c chmod -w ./ci/release/template/scripts/lib.sh diff --git a/ci/release/template/scripts/lib.sh b/ci/release/template/scripts/lib.sh index 0a232eb74..95119c42a 100644 --- a/ci/release/template/scripts/lib.sh +++ b/ci/release/template/scripts/lib.sh @@ -1,8 +1,21 @@ #!/bin/sh -if [ -n "${DEBUG-}" ]; then - set -x +# ************* +# DO NOT EDIT +# +# lib.sh was bundled together from +# +# - ./ci/sub/lib/rand.sh +# - ./ci/sub/lib/log.sh +# +# Generated by ./ci/release/gen_template_lib.sh. +# ************* + +#!/bin/sh +if [ "${LIB_RAND-}" ]; then + return 0 fi +LIB_RAND=1 rand() { seed="$1" @@ -14,15 +27,40 @@ rand() { } pick() { + if ! command -v shuf >/dev/null || ! command -v md5sum >/dev/null; then + eval "_echo \"\$3\"" + return + fi + seed="$1" shift i="$(rand "$seed" "1-$#")" eval "_echo \"\$$i\"" } +#!/bin/sh +if [ "${LIB_LOG-}" ]; then + return 0 +fi +LIB_LOG=1 + +if [ -n "${DEBUG-}" ]; then + set -x +fi + +export COLOR +if ! [ "${COLOR-}" = 0 -o "${COLOR-}" = false ]; then + if [ "${COLOR-}" = 1 -o "${COLOR-}" = true -o -t 1 ]; then + export _COLOR=1 + fi +fi tput() { - if [ -n "$TERM" ]; then - command tput "$@" + if [ "${COLOR-}" = 0 -o "${COLOR-}" = false ]; then + return 0 + fi + + if [ "${COLOR-}" = 1 -o "${COLOR-}" = true -o -t 1 ]; then + TERM=${TERM:-xterm-256color} command tput "$@" fi } @@ -59,10 +97,10 @@ printfp() {( prefix="$1" shift - if [ -z "${COLOR:-}" ]; then - COLOR="$(get_rand_color "$prefix")" + if [ -z "${FGCOLOR-}" ]; then + FGCOLOR="$(get_rand_color "$prefix")" fi - printf '%s' "$(setaf "$COLOR" "$prefix")" + printf '%s' "$(setaf "$FGCOLOR" "$prefix")" if [ $# -gt 0 ]; then printf ': ' @@ -74,13 +112,7 @@ catp() { prefix="$1" shift - printfp "$prefix" - printf ': ' - read -r line - _echo "$line" - - indent=$(repeat ' ' 2) - sed "s/^/$indent/" + sed "s/^/$(printfp "$prefix" '')/" } repeat() { @@ -94,48 +126,137 @@ strlen() { } echoerr() { - COLOR=1 echop err "$*" >&2 + FGCOLOR=1 echop err "$*" | humanpath>&2 } caterr() { - COLOR=1 catp err "$@" >&2 + FGCOLOR=1 catp err "$@" | humanpath >&2 } printferr() { - COLOR=1 printfp err "$@" >&2 + FGCOLOR=1 printfp err "$@" | humanpath >&2 } logp() { - echop "$@" >&2 + echop "$@" | humanpath >&2 } logfp() { - printfp "$@" >&2 + printfp "$@" | humanpath >&2 } logpcat() { - catp "$@" >&2 + catp "$@" | humanpath >&2 } log() { - COLOR=5 logp log "$@" + FGCOLOR=5 logp log "$@" } logf() { - COLOR=5 logfp log "$@" + FGCOLOR=5 logfp log "$@" } logcat() { - COLOR=5 catp log "$@" >&2 + FGCOLOR=5 logpcat log "$@" +} + +warn() { + FGCOLOR=3 logp warn "$@" +} + +warnf() { + FGCOLOR=3 logfp warn "$@" } sh_c() { - COLOR=3 logp exec "$*" + FGCOLOR=3 logp exec "$*" if [ -z "${DRY_RUN-}" ]; then - "$@" + eval "$@" + fi +} + +sudo_sh_c() { + if [ "$(id -u)" -eq 0 ]; then + sh_c "$@" + elif command -v doas >/dev/null; then + sh_c "doas $*" + elif command -v sudo >/dev/null; then + sh_c "sudo $*" + elif command -v su >/dev/null; then + sh_c "su root -c '$*'" + else + caterr <"$out" 2>&1 + code="$?" + set -e + if [ "$code" -eq 0 ]; then + return + fi + cat "$out" >&2 + return "$code" +} + +echo_dur() { + local dur=$1 + local h=$((dur/60/60)) + local m=$((dur/60%60)) + local s=$((dur%60)) + printf '%dh%dm%ds' "$h" "$m" "$s" +} + +sponge() { + dst="$1" + tmp="$(mktemp)" + cat > "$tmp" + cat "$tmp" > "$dst" +} + +stripansi() { + # First regex gets rid of standard xterm escape sequences for controlling + # visual attributes. + # The second regex I'm not 100% sure, the reference says it selects the US + # encoding but I'm not sure why that's necessary or why it always occurs + # in tput sgr0 before the standard escape sequence. + # See tput sgr0 | xxd + sed -e $'s/\x1b\[[0-9;]*m//g' -e $'s/\x1b(.//g' +} + +runtty() { + case "$(uname)" in + Darwin) + script -q /dev/null "$@" + ;; + Linux) + script -eqc "$*" + ;; + *) + echoerr "runtty: unsupported OS $(uname)" + return 1 + esac +} diff --git a/ci/sub b/ci/sub index df51b9089..c5e5d53ca 160000 --- a/ci/sub +++ b/ci/sub @@ -1 +1 @@ -Subproject commit df51b90892737ebe9feca3dd982bcdfc7f684834 +Subproject commit c5e5d53caa3241629a56b95eaf9780a68546ad61 diff --git a/go.sum b/go.sum index c9976a706..a7e3307a5 100644 --- a/go.sum +++ b/go.sum @@ -769,6 +769,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= oss.terrastruct.com/cmdlog v0.0.0-20221116181457-07977d95ac37 h1:Xy1JKJHc4hcuwi57s0BvGUY16GjxTtBmLUybsuGDU7E= +oss.terrastruct.com/cmdlog v0.0.0-20221116181457-07977d95ac37 h1:Xy1JKJHc4hcuwi57s0BvGUY16GjxTtBmLUybsuGDU7E= +oss.terrastruct.com/cmdlog v0.0.0-20221116181457-07977d95ac37/go.mod h1:ROL3yxl2X+S3O+Rls00qdX6aMh+p1dF8IdxDRwDDpsg= oss.terrastruct.com/cmdlog v0.0.0-20221116181457-07977d95ac37/go.mod h1:ROL3yxl2X+S3O+Rls00qdX6aMh+p1dF8IdxDRwDDpsg= oss.terrastruct.com/diff v1.0.2-0.20221116222035-8bf4dd3ab541 h1:I9B1O1IJ6spivIQxbFRZmbhAwVeLwrcQRR1JbYUOvrI= oss.terrastruct.com/diff v1.0.2-0.20221116222035-8bf4dd3ab541/go.mod h1:ags2QDy/T6jr69hT6bpmAmhr2H98n9o8Atf3QlUJPiU= diff --git a/install.sh b/install.sh index 5cc51950f..a9eb6bec3 100755 --- a/install.sh +++ b/install.sh @@ -53,9 +53,20 @@ if [ -n "${DEBUG-}" ]; then set -x fi +export COLOR +if ! [ "${COLOR-}" = 0 -o "${COLOR-}" = false ]; then + if [ "${COLOR-}" = 1 -o "${COLOR-}" = true -o -t 1 ]; then + export _COLOR=1 + fi +fi + tput() { - if [ -n "$TERM" ]; then - command tput "$@" + if [ "${COLOR-}" = 0 -o "${COLOR-}" = false ]; then + return 0 + fi + + if [ "${COLOR-}" = 1 -o "${COLOR-}" = true -o -t 1 ]; then + TERM=${TERM:-xterm-256color} command tput "$@" fi } @@ -92,10 +103,10 @@ printfp() {( prefix="$1" shift - if [ -z "${COLOR:-}" ]; then - COLOR="$(get_rand_color "$prefix")" + if [ -z "${FGCOLOR-}" ]; then + FGCOLOR="$(get_rand_color "$prefix")" fi - printf '%s' "$(setaf "$COLOR" "$prefix")" + printf '%s' "$(setaf "$FGCOLOR" "$prefix")" if [ $# -gt 0 ]; then printf ': ' @@ -121,15 +132,15 @@ strlen() { } echoerr() { - COLOR=1 echop err "$*" | humanpath>&2 + FGCOLOR=1 echop err "$*" | humanpath>&2 } caterr() { - COLOR=1 catp err "$@" | humanpath >&2 + FGCOLOR=1 catp err "$@" | humanpath >&2 } printferr() { - COLOR=1 printfp err "$@" | humanpath >&2 + FGCOLOR=1 printfp err "$@" | humanpath >&2 } logp() { @@ -145,27 +156,27 @@ logpcat() { } log() { - COLOR=5 logp log "$@" + FGCOLOR=5 logp log "$@" } logf() { - COLOR=5 logfp log "$@" + FGCOLOR=5 logfp log "$@" } logcat() { - COLOR=5 logpcat log "$@" + FGCOLOR=5 logpcat log "$@" } warn() { - COLOR=3 logp warn "$@" + FGCOLOR=3 logp warn "$@" } warnf() { - COLOR=3 logfp warn "$@" + FGCOLOR=3 logfp warn "$@" } sh_c() { - COLOR=3 logp exec "$*" + FGCOLOR=3 logp exec "$*" if [ -z "${DRY_RUN-}" ]; then eval "$@" fi @@ -280,9 +291,8 @@ LIB_FLAG=1 # FLAGSHIFT contains the number by which the arguments should be shifted to # start at the next flag/argument # -# After each call check $FLAG for the name of the parsed flag. -# If empty, then no more flags are left. -# Still, call shift "$FLAGSHIFT" in case there was a -- +# flag_parse exits with a non zero code when there are no more flags +# to be parsed. Still, call shift "$FLAGSHIFT" in case there was a -- # # If the argument for the flag is optional, then use ${FLAGARG-} to access # the argument if one was passed. Use ${FLAGARG+x} = x to check if it was set. @@ -310,18 +320,15 @@ flag_parse() { # Remove everything before first equal sign. FLAGARG="${1#*=}" FLAGSHIFT=1 + return 0 ;; -) - FLAG= - FLAGRAW= - unset FLAGARG FLAGSHIFT=0 + return 1 ;; --) - FLAG= - FLAGRAW= - unset FLAGARG FLAGSHIFT=1 + return 1 ;; -*) # Remove leading hyphens. @@ -343,15 +350,13 @@ flag_parse() { ;; esac fi + return 0 ;; *) - FLAG= - FLAGRAW= - unset FLAGARG FLAGSHIFT=0 + return 1 ;; esac - return 0 } flag_reqarg() { @@ -520,8 +525,7 @@ EOF main() { METHOD=standalone - while :; do - flag_parse "$@" + while flag_parse "$@"; do case "$FLAG" in h|help) help @@ -563,15 +567,12 @@ main() { flag_noarg && shift "$FLAGSHIFT" UNINSTALL=1 ;; - '') - shift "$FLAGSHIFT" - break - ;; *) flag_errusage "unrecognized flag $FLAGRAW" ;; esac done + shift "$FLAGSHIFT" if [ $# -gt 0 ]; then flag_errusage "no arguments are accepted" @@ -610,7 +611,7 @@ install() { TALA_VERSION="$( install_tala && echo "$VERSION" )" fi - COLOR=2 header success + FGCOLOR=2 header success log "d2-$VERSION-$OS-$ARCH has been successfully installed into $PREFIX" if [ -n "${TALA-}" ]; then log "tala-$TALA_VERSION-$OS-$ARCH has been successfully installed into $PREFIX"