install.sh: Get installs working

ci/release/README.md

@@ -14,6 +14,10 @@ it depends on from ../sub/lib.
 - ./release.sh is the top level script to generate a new release.
   Run with --help for usage.
 
+### upload_assets.sh
+
+- Called by ./release.sh to upload the assets for the release in $VERSION
+
 ## build.sh
 
 - ./build.sh builds the release archives for each platform into ./build/<VERSION>/*.tar.gz

ci/release/_build.sh

@@ -16,4 +16,7 @@ sh_c mkdir -p "$HW_BUILD_DIR/bin"
 sh_c go build -ldflags "'-X oss.terrastruct.com/d2/lib/version.Version=$VERSION'" \
   -o "$HW_BUILD_DIR/bin/d2" ./cmd/d2
 
-sh_c tar czf "$ARCHIVE" "$HW_BUILD_DIR"
+ARCHIVE=$PWD/$ARCHIVE
+cd "$(dirname "$HW_BUILD_DIR")"
+sh_c tar -czf "$ARCHIVE" "$(basename "$HW_BUILD_DIR")"
+cd ->/dev/null

ci/release/_install.sh

@@ -1,10 +1,11 @@
 #!/bin/sh
 set -eu
 
-cd -- "$(dirname "$0")/../.."
+cd -- "$(dirname "$0")/../sub/lib"
-. ./ci/sub/lib/log.sh
+. ./log.sh
-. ./ci/sub/lib/flag.sh
+. ./flag.sh
-cd -
+. ./release.sh
+cd - >/dev/null
 
 help() {
   arg0="$0"
@@ -13,7 +14,7 @@ help() {
   fi
 
   cat <<EOF
-usage: $arg0 [--dry-run] [--version vX.X.X] [--edge] [--method detect] [--prefix ~/.local]
+usage: $arg0 [--dry-run] [--version vX.X.X] [--edge] [--method detect] [--prefix /usr/local]
   [--tala] [--tala-version vX.X.X] [--force] [--uninstall]
 
 install.sh automates the installation of D2 onto your system. It currently only supports
@@ -28,7 +29,6 @@ Flags:
 
 --version vX.X.X
   Pass to have install.sh install the given version instead of the latest version.
-  note: currently unimplemented.
 
 --edge
   Pass to build and install D2 from source. This will still use --method if set to detect
@@ -40,16 +40,17 @@ Flags:
 --method [detect | standalone]
   Pass to control the method by which to install. Right now we only support standalone
   releases from GitHub but later we'll add support for brew, rpm, deb and more.
+  note: currently unimplemented.
 
   - detect is currently unimplemented but would use your OS's package manager
     automatically.
-  - standalone installs a standalone release archive into ~/.local
+  - standalone installs a standalone release archive into the unix hierarchy path
-     Add ~/.local/bin to your \$PATH to use it.
+     specified by --prefix which defaults to /usr/local
-     Control the unix hierarchy path with --prefix
+     Ensure /usr/local/bin is in your \$PATH to use it.
 
---prefix ~/.local
+--prefix /usr/local
   Controls the unix hierarchy path into which standalone releases are installed.
-  Defaults to ~/.local. You may also want to use /usr/local
+  Defaults to /usr/local. You may also want to use ~/.local to avoid needing sudo.
   Remember that whatever you use, you must have the bin directory of your prefix
   path in \$PATH to execute the d2 binary. For example, if my prefix directory is
   /usr/local then my \$PATH must contain /usr/local/bin.
@@ -60,7 +61,6 @@ Flags:
 
 --tala-version vX.X.X
   Install the passed version of tala instead of latest.
-  note: currently unimplemented.
 
 --force:
   Force installation over the existing version even if they match. It will attempt a clean
@@ -73,7 +73,7 @@ Flags:
   for uninstallation. With detect, the install script will try to use the OS package manager
   to uninstall instead.
 
-All downloaded assets are cached into ~/.cache/d2/install. use \$XDG_CACHE_HOME to change
+All downloaded assets are cached into ~/.cache/d2/release. use \$XDG_CACHE_HOME to change
 path of the cached assets.
 
 You can rerun install.sh to update your version of D2. install.sh will avoid reinstalling
@@ -82,6 +82,11 @@ EOF
 }
 
 main() {
+  if [ -n "${DEBUG-}" ]; then
+    set -x
+  fi
+
+  METHOD=standalone
   while :; do
     flag_parse "$@"
     case "$FLAG" in
@@ -96,14 +101,10 @@ main() {
       version)
         flag_nonemptyarg && shift "$FLAGSHIFT"
         VERSION=$FLAGARG
-        echoerr "$FLAGRAW is currently unimplemented"
-        exit 1
         ;;
       tala-version)
         flag_nonemptyarg && shift "$FLAGSHIFT"
         TALA_VERSION=$FLAGARG
-        echoerr "$FLAGRAW is currently unimplemented"
-        exit 1
         ;;
       edge)
         flag_noarg && shift "$FLAGSHIFT"
@@ -142,6 +143,125 @@ main() {
   if [ $# -gt 0 ]; then
     flag_errusage "no arguments are accepted"
   fi
+
+  if [ -n "${UNINSTALL-}" ]; then
+    uninstall
+    return 1
+  fi
+
+  REPO=${REPO:-terrastruct/d2}
+  PREFIX=${PREFIX:-/usr/local}
+  OS=$(os)
+  ARCH=$(arch)
+  CACHE_DIR=$(cache_dir)
+  mkdir -p "$CACHE_DIR"
+  VERSION=${VERSION:-latest}
+  if [ "$VERSION" = latest ]; then
+    VERSION=$(fetch_version_info)
+  fi
+
+  if command -v d2 >/dev/null; then
+    INSTALLED_VERSION="$(d2 version)"
+    if [ ! "${FORCE-}" -a "$VERSION" = "$INSTALLED_VERSION" ]; then
+      log "skipping installation as version $VERSION is already installed."
+      return 0
+    fi
+    log "uninstalling $INSTALLED_VERSION to install $VERSION"
+    # uninstall
+  fi
+
+  install_standalone
+}
+
+install_standalone() {
+  ARCHIVE="d2-$VERSION-$OS-$ARCH.tar.gz"
+  log "installing standalone release $ARCHIVE from github"
+
+  VERSION=$(fetch_version_info)
+  asset_line=$(cat "$CACHE_DIR/$VERSION.json" | grep -n "$ARCHIVE" | cut -d: -f1 | head -n1)
+  asset_url=$(sed -n $((asset_line-3))p "$CACHE_DIR/$VERSION.json" | sed 's/^.*: "\(.*\)",$/\1/g')
+  fetch_gh "$asset_url" "$CACHE_DIR/$ARCHIVE" 'application/octet-stream'
+
+  sh_c tar -C "$CACHE_DIR" -xzf "$CACHE_DIR/$ARCHIVE"
+  sh_c cd "$CACHE_DIR/d2-$VERSION"
+
+  sh_c="sh_c"
+  if !is_prefix_writable; then
+    sh_c="sudo_sh_c"
+  fi
+  "$sh_c" make install PREFIX="$PREFIX"
+}
+
+uninstall() {
+  log "uninstalling standalone release d2-$VERSION"
+
+  sh_c cd "$CACHE_DIR/d2-$VERSION"
+
+  sh_c="sh_c"
+  if !is_prefix_writable; then
+    sh_c="sudo_sh_c"
+  fi
+  "$sh_c" make uninstall PREFIX="$PREFIX"
+}
+
+is_prefix_writable() {
+  sh_c mkdir -p "$PREFIX" 2>/dev/null || true
+  # The reason for checking whether bin is writable specifically is that on macOS you have
+  # /usr/local owned by root but you don't need root to write to its subdirectories which
+  # is all we want to do.
+  if [ ! -w "$PREFIX/bin" ]; then
+    return 0
+  fi
+}
+
+cache_dir() {
+  if [ -n "${XDG_CACHE_HOME-}" ]; then
+    echo "$XDG_CACHE_HOME/d2/release"
+  elif [ -n "${HOME-}" ]; then
+    echo "$HOME/.cache/d2/release"
+  else
+    echo "/tmp/d2-cache/release"
+  fi
+}
+
+fetch_version_info() {
+  req_version=$VERSION
+  log "fetching info on version $req_version"
+  if [ -e "$CACHE_DIR/$req_version.json" ]; then
+    log "reusing $CACHE_DIR/$req_version.json"
+  fi
+
+  rm -f "$CACHE_DIR/req_version.json"
+  if [ "$req_version" = latest ]; then
+    release_info_url="https://api.github.com/repos/$REPO/releases/$req_version"
+  else
+    release_info_url="https://api.github.com/repos/$REPO/releases/tags/$req_version"
+  fi
+  fetch_gh "$release_info_url" "$CACHE_DIR/$req_version.json" \
+    'application/json'
+  VERSION=$(cat "$CACHE_DIR/$req_version.json" | grep -m1 tag_name | sed 's/^.*: "\(.*\)",$/\1/g')
+  if [ "$req_version" = latest ]; then
+    mv "$CACHE_DIR/$req_version.json" "$CACHE_DIR/$VERSION.json"
+  fi
+  echo "$VERSION"
+}
+
+curl_gh() {
+  sh_c curl -fL ${GITHUB_TOKEN+"-H \"Authorization: Bearer \$GITHUB_TOKEN\""} "$@"
+}
+
+fetch_gh() {
+  url=$1
+  file=$2
+  accept=$3
+
+  if [ -e "$file" ]; then
+    log "reusing $file"
+    return
+  fi
+
+  curl_gh -#o "$file.inprogress" -C- -H "'Accept: $accept'" "$url"
+  sh_c mv "$file.inprogress" "$file"
 }
 
 main "$@"

ci/release/build.sh

@@ -7,7 +7,7 @@ help() {
   cat <<EOF
 usage: $0 [--rebuild] [--local] [--dry-run] [--run=regex] [--host-only]
 
-$0 builds D2 release archives into ./ci/release/build/<version>/d2-<version>.tar.gz
+$0 builds D2 release archives into ./ci/release/build/<version>/d2-<VERSION>-<OS>-<ARCH>.tar.gz
 
 The version is detected via git describe which will use the git tag for the current
 commit if available.
@@ -32,12 +32,13 @@ Flags:
 --run=regex
   Use to run only the OS-ARCH jobs that match the given regex. e.g. --run=linux only runs
   the linux jobs. --run=linux-amd64 only runs the linux-amd64 job.
+
+--version vX.X.X
+  Use to overwrite the version detected from git.
 EOF
 }
 
 main() {
-  VERSION="$(git_describe_ref)"
-  BUILD_DIR="ci/release/build/$VERSION"
   while :; do
     flag_parse "$@"
     case "$FLAG" in
@@ -64,6 +65,11 @@ main() {
       host-only)
         flag_noarg && shift "$FLAGSHIFT"
         HOST_ONLY=1
+        LOCAL=1
+        ;;
+      version)
+        flag_nonemptyarg && shift "$FLAGSHIFT"
+        VERSION=$FLAGARG
         ;;
       '')
         shift "$FLAGSHIFT"
@@ -79,6 +85,8 @@ main() {
     flag_errusage "no arguments are accepted"
   fi
 
+  VERSION=${VERSION:-$(git_describe_ref)}
+  BUILD_DIR=ci/release/build/$VERSION
   if [ -n "${HOST_ONLY-}" ]; then
     runjob $(os)-$(arch) "OS=$(os) ARCH=$(arch) build" &
     waitjobs
@@ -93,8 +101,8 @@ main() {
 }
 
 build() {
-  HW_BUILD_DIR="$BUILD_DIR/$OS/$ARCH/d2-$VERSION"
+  HW_BUILD_DIR="$BUILD_DIR/$OS-$ARCH/d2-$VERSION"
-  ARCHIVE="$BUILD_DIR/d2-$OS-$ARCH-$VERSION.tar.gz"
+  ARCHIVE="$BUILD_DIR/d2-$VERSION-$OS-$ARCH.tar.gz"
 
   if [ -e "$ARCHIVE" -a -z "${REBUILD-}" ]; then
     log "skipping as already built at $ARCHIVE"

ci/release/gen_install.sh

@@ -30,6 +30,7 @@ sh_c cat \
   ./ci/sub/lib/rand.sh \
   ./ci/sub/lib/log.sh \
   ./ci/sub/lib/flag.sh \
+  ./ci/sub/lib/release.sh \
   \| sed "-e'/^\. /d'" \>\> ./install.sh
 sh_c cat ./ci/release/_install.sh \
   \| sed -n "'/cd -- \"\$(dirname/,/cd -/!p'" \>\> install.sh

ci/release/release.sh

@@ -21,6 +21,8 @@ Flags:
                changelogs/v0.0.99-alpha.1.md. This is because you want to maintain the
                changelog entries for the eventual final release.
 --dry-run: Print the commands that would be ran without executing them.
+--skip-build: Skip the build in case you want to upload your own specific assets.
+              Mainly for testing and debugging.
 
 Process:
 
@@ -82,6 +84,10 @@ main() {
         flag_noarg && shift "$FLAGSHIFT"
         DRY_RUN=1
         ;;
+      skip-build)
+        flag_noarg && shift "$FLAGSHIFT"
+        SKIP_BUILD=1
+        ;;
       '')
         shift "$FLAGSHIFT"
         break
@@ -185,15 +191,19 @@ _7_ensure_pr() {
     return 0
   fi
 
-  pr_url="$(sh_c gh pr create --fill --body "$body" | tee /dev/stderr)"
+  pr_url="$(sh_c gh pr create --fill --body "'$body'" | tee /dev/stderr)"
 }
 
 _8_ensure_assets() {
+  if [ "${SKIP_BUILD-}" ]; then
+    warn "skipping building of assets due to --skip-build"
+    return 0
+  fi
   sh_c ./ci/release/build.sh ${REBUILD:+--rebuild}
 }
 
 _9_upload_assets() {
-  sh_c gh release upload --clobber "$VERSION" "./ci/release/build/$VERSION"/*.tar.gz
+  ./ci/release/upload_assets.sh $VERSION
 }
 
 main "$@"

ci/release/template/Makefile

@@ -5,8 +5,8 @@ PREFIX = $(DESTDIR)/usr/local
 
 .PHONY: install
 install:
-	./scripts/install.sh
+	PREFIX='$(PREFIX)' ./scripts/install.sh
 
 .PHONY: uninstall
 uninstall:
-	./scripts/uninstall.sh
+	PREFIX='$(PREFIX)' ./scripts/uninstall.sh

ci/release/upload_assets.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -eu
+cd -- "$(dirname "$0")/../.."
+. ./ci/sub/lib.sh
+
+help() {
+  cat <<EOF
+usage: $0 <version>
+
+Uploads the assets for release <version> to GitHub.
+
+For example, if <version> is v0.0.99 then it uploads files matching
+./ci/release/build/v0.0.99/*.tar.gz to the GitHub release v0.0.99.
+
+Example:
+  $0 v0.0.99
+EOF
+}
+
+main() {
+  while :; do
+    flag_parse "$@"
+    case "$FLAG" in
+      h|help)
+        help
+        return 0
+        ;;
+      '')
+        shift "$FLAGSHIFT"
+        break
+        ;;
+      *)
+        flag_errusage "unrecognized flag $FLAGRAW"
+        ;;
+    esac
+  done
+
+  if [ $# -ne 1 ]; then
+    flag_errusage "first argument must be release version like v0.0.99"
+  fi
+  VERSION="$1"
+  shift
+
+  sh_c gh release upload --clobber "$VERSION" "./ci/release/build/$VERSION"/*.tar.gz
+}
+
+main "$@"

ci/sub

@@ -1 +1 @@
-Subproject commit 81e033b7702131e917a16396b44c20512d29029b
+Subproject commit 5a1c8e5e36a0b0c4b0d5e2ad4e08c3c815dd4bea

install.sh

@@ -499,6 +499,13 @@ main() {
   if [ $# -gt 0 ]; then
     flag_errusage "no arguments are accepted"
   fi
+
+  REPO=${REPO:-terrastruct/d2}
+  latest_version
+}
+
+latest_version() {
+  curl -fsSLI -o/dev/null -w '%{url_effective}' "https://github.com/$REPO/latest"
 }
 
 main "$@"

lib/version/version.go

@@ -2,6 +2,7 @@ package version
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/google/go-github/github"
 	"oss.terrastruct.com/cmdlog"
@@ -11,12 +12,17 @@ import (
 var Version = "master (built from source)"
 
 func CheckVersion(ctx context.Context, logger *cmdlog.Logger) {
-	logger.Info.Printf("D2 version: %s\n", Version)
+	fmt.Println(Version)
 
 	if Version == "master (built from source)" {
 		return
 	}
 
+	// Install script uses -v to check the version, we shouldn't be checking for
+	// updates here...
+	// https://github.com/terrastruct/d2/issues/49#issuecomment-1313229683
+	return
+
 	logger.Info.Printf("Checking for updates...")
 	latest, err := getLatestVersion(ctx)
 	if err != nil {