diff --git a/.dockerignore b/.dockerignore
index eefa27e4..06a2da46 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -12,7 +12,6 @@ test/
 .gitmodules
 appveyor.yml
 CHANGES.md
-deps.edn
 Dockerfile
 LICENSE
 README.md
diff --git a/Dockerfile b/Dockerfile
index e01c8911..6b4aef4f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,16 +1,18 @@
-FROM clojure:lein-2.9.1 AS BASE
+FROM clojure:openjdk-11-lein-2.9.6-bullseye AS BASE
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update
-RUN apt install --no-install-recommends -yy curl unzip build-essential zlib1g-dev sudo
+RUN apt install --no-install-recommends -yy build-essential zlib1g-dev
 WORKDIR "/opt"
-RUN curl -sLO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-21.2.0/graalvm-ce-java11-linux-amd64-21.2.0.tar.gz
-RUN tar -xzf graalvm-ce-java11-linux-amd64-21.2.0.tar.gz
+
+ENV GRAALVM_VERSION="21.2.0"
+RUN curl -sLO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${GRAALVM_VERSION}/graalvm-ce-java11-linux-amd64-${GRAALVM_VERSION}.tar.gz
+RUN tar -xzf graalvm-ce-java11-linux-amd64-${GRAALVM_VERSION}.tar.gz
 
 ARG BABASHKA_XMX="-J-Xmx4500m"
 
-ENV GRAALVM_HOME="/opt/graalvm-ce-java11-21.2.0"
-ENV JAVA_HOME="/opt/graalvm-ce-java11-21.2.0/bin"
+ENV GRAALVM_HOME="/opt/graalvm-ce-java11-${GRAALVM_VERSION}"
+ENV JAVA_HOME="/opt/graalvm-ce-java11-${GRAALVM_VERSION}/bin"
 ENV PATH="$JAVA_HOME:$PATH"
 ENV BABASHKA_XMX=$BABASHKA_XMX
 
diff --git a/script/setup-musl b/script/setup-musl
index 921c6d7f..b3dfc6ac 100755
--- a/script/setup-musl
+++ b/script/setup-musl
@@ -12,29 +12,7 @@ if [[ -z "${BABASHKA_MUSL:-}" ]]; then
     exit 0
 fi
 
-# This script sets up the latest available musl-tools using apt pinning from Debian bullseye (11).
-# The one available in stretch (Debian 9) is outdated and this ensures we get the latest improvements
-# This explictly installs musl from bullseye and keeps the others at a higher priority
-# TODO: remove after clojure image is updated to bullseye
-
-cat >> /etc/apt/sources.list <<eof
-deb http://ftp.us.debian.org/debian bullseye main non-free contrib
-deb http://non-us.debian.org/debian-non-us bullseye/non-us main contrib non-free
-eof
-
-cat >> /etc/apt/preferences <<eof
-Package: *
-Pin: release a=stable
-Pin-Priority: 700
-Package: *
-Pin: release a=testing
-Pin-Priority: 650
-Package: *
-pin: release a=bullseye
-pin-priority: 600
-eof
-
-apt-get update -y && apt-get install musl-tools/bullseye -y
+apt-get update -y && apt-get install musl-tools -y
 
 ZLIB_VERSION="1.2.11"