From 9ca3a3bd317eb7bf076f48d28b0477c43bcf70f0 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Mon, 13 Sep 2021 18:28:13 -0300 Subject: [PATCH] Use Debian bullseye on CI/CD/Dockerfile (#1005) * Migrate `Dockerfile` to use a bullseye-based image Also, some QoL changes on the `Dockerfile`: - Parametrize GRAALVM_VERSION to make it easier to update - Remove installing programs already included on base image - Remove `deps.edn` from `.dockerignore` since it is necessary to build * Migrate CircleCI to use a bullseye-based image Also some modifications necessary to make it work: - Migrate from `circleci` images to `clojure` since they're out-of-date and also seem to be considered legacy: https://circleci.com/docs/2.0/circleci-images/#legacy-language-images - Remove unnecessary usage of `sudo`, since all commands runs as root - Sync packages with `Dockerfile`. This will make easier to test locally if everything will work (of course, it is not 100% guarantee) * Remove lsof * Remove "Install {Clojure,Leiningen}" steps Already included on the base image. * Do not change directory when downloading GraalVM * Move "Download GraalVM" to script/download-graalvm * Set GRAALVM_HOME correctly * Unbreak mac build * Revert "Set GRAALVM_HOME correctly" This reverts commit 5e2a6158dc28958bda7c2bd58c5597ebe02d6aef. * Set GRAALVM_HOME correctly, take 2 * Improve download-graalvm script * Re-added "Install Clojure" steps * Set amd64 as default GRAALVM_ARCH * Unbreak linux-aarch64 * Ubreak jvm * Do not change directory * Fix yaml * Fix mac/linux-aarch64 * Add missing Clojure * Fix cache * Move GraalVM installation to /tmp * Use script/install-clojure * Use /opt instead of /tmp to install GraalVM * Revert "Use /opt instead of /tmp to install GraalVM" This reverts commit 3cfad03c8ed641bb915ea19953423c43e801eafc. * Use CircleCI's Clojure images again * Go back to installing GraalVM on $HOME --- .circleci/config.yml | 119 +++++++++++------------------------------ .dockerignore | 1 - Dockerfile | 14 ++--- script/install-clojure | 14 +++-- script/install-graalvm | 37 +++++++++++++ script/setup-musl | 24 +-------- 6 files changed, 87 insertions(+), 122 deletions(-) create mode 100755 script/install-graalvm diff --git a/.circleci/config.yml b/.circleci/config.yml index 9118701b..9fd8ab7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,8 +6,7 @@ version: 2.1 jobs: jvm: docker: - # specify the version you desire here - - image: circleci/clojure:lein-2.9.1 + - image: circleci/clojure:openjdk-11-lein-2.9.6-bullseye working_directory: ~/repo environment: LEIN_ROOT: "true" @@ -28,13 +27,7 @@ jobs: - run: name: Install Clojure command: | - wget -nc https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh - - run: - name: Install lsof - command: | - sudo apt-get install lsof + sudo script/install-clojure - run: name: Run JVM tests command: | @@ -72,10 +65,11 @@ jobs: key: v1-dependencies-{{ checksum "project.clj" }}-{{ checksum "deps.edn" }} linux: docker: - - image: circleci/clojure:lein-2.9.1 + - image: circleci/clojure:openjdk-11-lein-2.9.6-bullseye working_directory: ~/repo environment: LEIN_ROOT: "true" + GRAALVM_VERSION: "21.2.0" GRAALVM_HOME: /home/circleci/graalvm-ce-java11-21.2.0 BABASHKA_PLATFORM: linux # used in release script BABASHKA_TEST_ENV: native @@ -94,26 +88,16 @@ jobs: - run: name: Install Clojure command: | - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh - - run: - name: Install lsof - command: | - sudo apt-get install lsof + sudo script/install-clojure - run: name: Install native dev tools command: | sudo apt-get update - sudo apt-get -y install g++ zlib1g-dev + sudo apt-get -y install build-essential zlib1g-dev - run: name: Download GraalVM command: | - cd ~ - if ! [ -d graalvm-ce-java11-21.2.0 ]; then - curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.2.0/graalvm-ce-java11-linux-amd64-21.2.0.tar.gz - tar xzf graalvm-ce-java11-linux-amd64-21.2.0.tar.gz - fi + script/install-graalvm - run: name: Build binary command: | @@ -147,10 +131,11 @@ jobs: ./bb .circleci/script/publish_artifact.clj || true linux-static: docker: - - image: circleci/clojure:lein-2.9.1 + - image: circleci/clojure:openjdk-11-lein-2.9.6-bullseye working_directory: ~/repo environment: LEIN_ROOT: "true" + GRAALVM_VERSION: "21.2.0" GRAALVM_HOME: /home/circleci/graalvm-ce-java11-21.2.0 BABASHKA_PLATFORM: linux # used in release script BABASHKA_TEST_ENV: native @@ -181,27 +166,17 @@ jobs: - run: name: Install Clojure command: | - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh - - run: - name: Install lsof - command: | - sudo apt-get install lsof + sudo script/install-clojure - run: name: Install native dev tools command: | sudo apt-get update - sudo apt-get -y install gcc g++ zlib1g-dev + sudo apt-get -y install build-essential zlib1g-dev sudo -E script/setup-musl - run: name: Download GraalVM command: | - cd ~ - if ! [ -d graalvm-ce-java11-21.2.0 ]; then - curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.2.0/graalvm-ce-java11-linux-amd64-21.2.0.tar.gz - tar xzf graalvm-ce-java11-linux-amd64-21.2.0.tar.gz - fi + script/install-graalvm - run: name: Build binary command: | @@ -241,6 +216,7 @@ jobs: working_directory: ~/repo environment: LEIN_ROOT: "true" + GRAALVM_VERSION: "21.2.0" GRAALVM_HOME: /home/circleci/graalvm-ce-java11-21.2.0 BABASHKA_PLATFORM: linux # used in release script BABASHKA_ARCH: aarch64 @@ -261,36 +237,22 @@ jobs: then circleci task halt fi + - run: + name: Install Clojure + command: | + sudo script/install-clojure - restore_cache: keys: - linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Leiningen - command: | - sudo script/install-leiningen - - run: - name: Install Clojure - command: | - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh - - run: - name: Install lsof - command: | - sudo apt-get install lsof - run: name: Install native dev tools command: | sudo apt-get update - sudo apt-get -y install gcc g++ zlib1g-dev + sudo apt-get -y install build-essential zlib1g-dev - run: name: Download GraalVM command: | - cd ~ - if ! [ -d graalvm-ce-java11-21.2.0 ]; then - curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.2.0/graalvm-ce-java11-linux-aarch64-21.2.0.tar.gz - tar xzf graalvm-ce-java11-linux-aarch64-21.2.0.tar.gz - fi + script/install-graalvm - run: name: Build binary command: | @@ -330,6 +292,7 @@ jobs: working_directory: ~/repo environment: LEIN_ROOT: "true" + GRAALVM_VERSION: "21.2.0" GRAALVM_HOME: /home/circleci/graalvm-ce-java11-21.2.0 BABASHKA_PLATFORM: linux # used in release script BABASHKA_ARCH: aarch64 @@ -351,37 +314,23 @@ jobs: # then # circleci task halt # fi + - run: + name: Install Clojure + command: | + sudo script/install-clojure - restore_cache: keys: - linux-aarch64-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - - run: - name: Install Leiningen - command: | - sudo script/install-leiningen - - run: - name: Install Clojure - command: | - wget https://download.clojure.org/install/linux-install-1.10.1.447.sh - chmod +x linux-install-1.10.1.447.sh - sudo ./linux-install-1.10.1.447.sh - - run: - name: Install lsof - command: | - sudo apt-get install lsof - run: name: Install native dev tools command: | sudo apt-get update - sudo apt-get -y install gcc g++ zlib1g-dev + sudo apt-get -y install build-essential zlib1g-dev # sudo -E script/setup-musl - run: name: Download GraalVM command: | - cd ~ - if ! [ -d graalvm-ce-java11-21.2.0 ]; then - curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.2.0/graalvm-ce-java11-linux-aarch64-21.2.0.tar.gz - tar xzf graalvm-ce-java11-linux-aarch64-21.2.0.tar.gz - fi + script/install-graalvm - run: name: Build binary command: | @@ -418,6 +367,7 @@ jobs: xcode: "12.0.0" environment: MACOSX_DEPLOYMENT_TARGET: 10.13 # 10.12 is EOL + GRAALVM_VERSION: "21.2.0" GRAALVM_HOME: /Users/distiller/graalvm-ce-java11-21.2.0/Contents/Home BABASHKA_PLATFORM: macos # used in release script BABASHKA_TEST_ENV: native @@ -436,7 +386,7 @@ jobs: - run: name: Install Clojure command: | - script/install-clojure /usr/local + script/install-clojure - run: name: Install Leiningen command: | @@ -444,12 +394,7 @@ jobs: - run: name: Download GraalVM command: | - cd ~ - ls -la - if ! [ -d graalvm-ce-java11-21.2.0 ]; then - curl -O -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.2.0/graalvm-ce-java11-darwin-amd64-21.2.0.tar.gz - tar xzf graalvm-ce-java11-darwin-amd64-21.2.0.tar.gz - fi + script/install-graalvm - run: name: Build binary command: | @@ -470,7 +415,7 @@ jobs: - save_cache: paths: - ~/.m2 - - ~/graalvm-ce-java11-21.2.0 + - ~/graalvm-ce-java11-21.2.0/Contents/Home key: mac-{{ checksum "project.clj" }}-{{ checksum ".circleci/config.yml" }} - store_artifacts: path: /tmp/release @@ -482,7 +427,7 @@ jobs: deploy: resource_class: large docker: - - image: circleci/clojure:lein-2.9.1 + - image: circleci/clojure:lein-2.9.6 working_directory: ~/repo environment: LEIN_ROOT: "true" @@ -505,7 +450,7 @@ jobs: key: v1-dependencies-{{ checksum "project.clj" }} docker: docker: - - image: circleci/buildpack-deps:stretch + - image: circleci/buildpack-deps:bullseye steps: - checkout - attach_workspace: diff --git a/.dockerignore b/.dockerignore index eefa27e4..06a2da46 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,7 +12,6 @@ test/ .gitmodules appveyor.yml CHANGES.md -deps.edn Dockerfile LICENSE README.md diff --git a/Dockerfile b/Dockerfile index e01c8911..6b4aef4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,18 @@ -FROM clojure:lein-2.9.1 AS BASE +FROM clojure:openjdk-11-lein-2.9.6-bullseye AS BASE ENV DEBIAN_FRONTEND=noninteractive RUN apt update -RUN apt install --no-install-recommends -yy curl unzip build-essential zlib1g-dev sudo +RUN apt install --no-install-recommends -yy build-essential zlib1g-dev WORKDIR "/opt" -RUN curl -sLO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.2.0/graalvm-ce-java11-linux-amd64-21.2.0.tar.gz -RUN tar -xzf graalvm-ce-java11-linux-amd64-21.2.0.tar.gz + +ENV GRAALVM_VERSION="21.2.0" +RUN curl -sLO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${GRAALVM_VERSION}/graalvm-ce-java11-linux-amd64-${GRAALVM_VERSION}.tar.gz +RUN tar -xzf graalvm-ce-java11-linux-amd64-${GRAALVM_VERSION}.tar.gz ARG BABASHKA_XMX="-J-Xmx4500m" -ENV GRAALVM_HOME="/opt/graalvm-ce-java11-21.2.0" -ENV JAVA_HOME="/opt/graalvm-ce-java11-21.2.0/bin" +ENV GRAALVM_HOME="/opt/graalvm-ce-java11-${GRAALVM_VERSION}" +ENV JAVA_HOME="/opt/graalvm-ce-java11-${GRAALVM_VERSION}/bin" ENV PATH="$JAVA_HOME:$PATH" ENV BABASHKA_XMX=$BABASHKA_XMX diff --git a/script/install-clojure b/script/install-clojure index f4781ba1..138e483f 100755 --- a/script/install-clojure +++ b/script/install-clojure @@ -1,10 +1,14 @@ #!/usr/bin/env bash -install_dir=${1:-/usr/local} +set -euo pipefail + +CLOJURE_TOOLS_VERSION="1.10.1.447" + +install_dir="${1:-/usr/local}" mkdir -p "$install_dir" cd /tmp -curl -O -sL https://download.clojure.org/install/clojure-tools-1.10.1.447.tar.gz -tar xzf clojure-tools-1.10.1.447.tar.gz +curl -O -sL "https://download.clojure.org/install/clojure-tools-$CLOJURE_TOOLS_VERSION.tar.gz" +tar xzf "clojure-tools-$CLOJURE_TOOLS_VERSION.tar.gz" cd clojure-tools clojure_lib_dir="$install_dir/lib/clojure" mkdir -p "$clojure_lib_dir/libexec" @@ -18,6 +22,6 @@ cp clojure "$install_dir/bin" cp clj "$install_dir/bin" cd /tmp -rm -rf clojure-tools-1.10.1.447.tar.gz -rm -rf clojure-tools +rm -rf "clojure-tools-$CLOJURE_TOOLS_VERSION.tar.gz" +rm -rf "clojure-tools" echo "Installed clojure to $install_dir/bin" diff --git a/script/install-graalvm b/script/install-graalvm new file mode 100755 index 00000000..19d43c5a --- /dev/null +++ b/script/install-graalvm @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +INSTALL_DIR="${1:-$HOME}" + +GRAALVM_VERSION="${GRAALVM_VERSION:-21.2.0}" + +case "$BABASHKA_PLATFORM" in + macos) + GRAALVM_PLATFORM="darwin" + ;; + linux) + GRAALVM_PLATFORM="linux" + ;; +esac + +case "${BABASHKA_ARCH:-}" in + aarch64) + GRAALVM_ARCH="aarch64" + ;; + *) + GRAALVM_ARCH="amd64" + ;; +esac + +GRAALVM_FILENAME="graalvm-ce-java11-$GRAALVM_PLATFORM-$GRAALVM_ARCH-$GRAALVM_VERSION.tar.gz" + +pushd "$INSTALL_DIR" >/dev/null + +if ! [ -d "graalvm-ce-java11-$GRAALVM_VERSION" ]; then + echo "Downloading GraalVM $GRAALVM_PLATFORM-$GRAALVM_ARCH-$GRAALVM_VERSION on '$PWD'..." + curl -O -sL "https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-$GRAALVM_VERSION/$GRAALVM_FILENAME" + tar xzf "$GRAALVM_FILENAME" +fi + +popd >/dev/null diff --git a/script/setup-musl b/script/setup-musl index 921c6d7f..b3dfc6ac 100755 --- a/script/setup-musl +++ b/script/setup-musl @@ -12,29 +12,7 @@ if [[ -z "${BABASHKA_MUSL:-}" ]]; then exit 0 fi -# This script sets up the latest available musl-tools using apt pinning from Debian bullseye (11). -# The one available in stretch (Debian 9) is outdated and this ensures we get the latest improvements -# This explictly installs musl from bullseye and keeps the others at a higher priority -# TODO: remove after clojure image is updated to bullseye - -cat >> /etc/apt/sources.list <> /etc/apt/preferences <