src/ci/docker/run.sh - toolchain/rustc - Git at Google

 #!/usr/bin/env bash

 set -e

 export MSYS_NO_PATHCONV=1

 script=`cd $(dirname $0) && pwd`/`basename $0`

 image=""
 dev=0

 while [[ $# -gt 0 ]]
 do
   case "$1" in
     --dev)
       dev=1
       ;;
     *)
       if [ -n "$image" ]
       then
         echo "expected single argument for the image name"
         exit 1
       fi
       image="$1"
       ;;
   esac
   shift
 done

 script_dir="`dirname $script`"
 docker_dir="${script_dir}/host-$(uname -m)"
 ci_dir="`dirname $script_dir`"
 src_dir="`dirname $ci_dir`"
 root_dir="`dirname $src_dir`"

 objdir=$root_dir/obj
 dist=$objdir/build/dist

 source "$ci_dir/shared.sh"

 CACHE_DOMAIN="${CACHE_DOMAIN:-ci-caches.rust-lang.org}"

 if [ -f "$docker_dir/$image/Dockerfile" ]; then
     if [ "$CI" != "" ]; then
       hash_key=/tmp/.docker-hash-key.txt
       rm -f "${hash_key}"
       echo $image >> $hash_key

       cat "$docker_dir/$image/Dockerfile" >> $hash_key
       # Look for all source files involves in the COPY command
       copied_files=/tmp/.docker-copied-files.txt
       rm -f "$copied_files"
       for i in $(sed -n -e '/^COPY --from=/! s/^COPY \(.*\) .*$/\1/p' \
           "$docker_dir/$image/Dockerfile"); do
         # List the file names
         find "$script_dir/$i" -type f >> $copied_files
       done
       # Sort the file names and cat the content into the hash key
       sort $copied_files | xargs cat >> $hash_key

       # Include the architecture in the hash key, since our Linux CI does not
       # only run in x86_64 machines.
       uname -m >> $hash_key

       docker --version >> $hash_key

       # Include cache version. Can be used to manually bust the Docker cache.
       echo "2" >> $hash_key

       cksum=$(sha512sum $hash_key | \
         awk '{print $1}')

       url="https://$CACHE_DOMAIN/docker/$cksum"

       echo "Attempting to download $url"
       rm -f /tmp/rustci_docker_cache
       set +e
       retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \
         -o /tmp/rustci_docker_cache "$url"

       docker_archive_hash=$(sha512sum /tmp/rustci_docker_cache | awk '{print $1}')
       echo "Downloaded archive hash: ${docker_archive_hash}"

       echo "Loading images into docker"
       # docker load sometimes hangs in the CI, so time out after 10 minutes with TERM,
       # KILL after 12 minutes
       loaded_images=$(/usr/bin/timeout -k 720 600 docker load -i /tmp/rustci_docker_cache \
         | sed 's/.* sha/sha/')
       set -e
       printf "Downloaded containers:\n$loaded_images\n"
     fi

     dockerfile="$docker_dir/$image/Dockerfile"
     if [ -x /usr/bin/cygpath ]; then
         context="`cygpath -w $script_dir`"
         dockerfile="`cygpath -w $dockerfile`"
     else
         context="$script_dir"
     fi
     echo "::group::Building docker image for $image"

     # As of August 2023, Github Actions have updated Docker to 23.X,
     # which uses the BuildKit by default. It currently throws aways all
     # intermediate layers, which breaks our usage of S3 layer caching.
     # Therefore we opt-in to the old build backend for now.
     export DOCKER_BUILDKIT=0
     retry docker \
       build \
       --rm \
       -t rust-ci \
       -f "$dockerfile" \
       "$context"
     echo "::endgroup::"

     if [ "$CI" != "" ]; then
       s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
       upload="aws s3 cp - $s3url"
       digest=$(docker inspect rust-ci --format '{{.Id}}')
       echo "Built container $digest"
       if ! grep -q "$digest" <(echo "$loaded_images"); then
         echo "Uploading finished image $digest to $url"
         set +e
         # Print image history for easier debugging of layer SHAs
         docker history rust-ci
         docker history -q rust-ci | \
           grep -v missing | \
           xargs docker save | \
           gzip | \
           $upload
         set -e
       else
         echo "Looks like docker image is the same as before, not uploading"
       fi
       # Record the container image for reuse, e.g. by rustup.rs builds
       info="$dist/image-$image.txt"
       mkdir -p "$dist"
       echo "$url" >"$info"
       echo "$digest" >>"$info"
       cat "$info"
     fi
 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
     if isCI; then
         echo Cannot run disabled images on CI!
         exit 1
     fi
     # Transform changes the context of disabled Dockerfiles to match the enabled ones
     tar --transform 's#disabled/#./#' -C $script_dir -c . | docker \
       build \
       --rm \
       -t rust-ci \
       -f "host-$(uname -m)/$image/Dockerfile" \
       -
 else
     echo Invalid image: $image

     # Check whether the image exists for other architectures
     for arch_dir in "${script_dir}"/host-*; do
         # Avoid checking non-directories and the current host architecture directory
         if ! [[ -d "${arch_dir}" ]]; then
             continue
         fi
         if [[ "${arch_dir}" = "${docker_dir}" ]]; then
             continue
         fi

         arch_name="$(basename "${arch_dir}" | sed 's/^host-//')"
         if [[ -f "${arch_dir}/${image}/Dockerfile" ]]; then
             echo "Note: the image exists for the ${arch_name} host architecture"
         elif [[ -f "${arch_dir}/disabled/${image}/Dockerfile" ]]; then
             echo "Note: the disabled image exists for the ${arch_name} host architecture"
         else
             continue
         fi
         echo "Note: the current host architecture is $(uname -m)"
     done

     exit 1
 fi

 mkdir -p $HOME/.cargo
 mkdir -p $objdir/tmp
 mkdir -p $objdir/cores
 mkdir -p /tmp/toolstate

 args=
 if [ "$SCCACHE_BUCKET" != "" ]; then
     args="$args --env SCCACHE_BUCKET"
     args="$args --env SCCACHE_REGION"
     args="$args --env AWS_ACCESS_KEY_ID"
     args="$args --env AWS_SECRET_ACCESS_KEY"
     args="$args --env AWS_REGION"
 else
     mkdir -p $HOME/.cache/sccache
     args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache"
 fi

 # Run containers as privileged as it should give them access to some more
 # syscalls such as ptrace and whatnot. In the upgrade to LLVM 5.0 it was
 # discovered that the leak sanitizer apparently needs these syscalls nowadays so
 # we'll need `--privileged` for at least the `x86_64-gnu` builder, so this just
 # goes ahead and sets it for all builders.
 args="$args --privileged"

 # Things get a little weird if this script is already running in a docker
 # container. If we're already in a docker container then we assume it's set up
 # to do docker-in-docker where we have access to a working `docker` command.
 #
 # If this is the case (we check via the presence of `/.dockerenv`)
 # then we can't actually use the `--volume` argument. Typically we use
 # `--volume` to efficiently share the build and source directory between this
 # script and the container we're about to spawn. If we're inside docker already
 # though the `--volume` argument maps the *host's* folder to the container we're
 # about to spawn, when in fact we want the folder in this container itself. To
 # work around this we use a recipe cribbed from
 # https://circleci.com/docs/2.0/building-docker-images/#mounting-folders to
 # create a temporary container with a volume. We then copy the entire source
 # directory into this container, and then use that copy in the container we're
 # about to spawn. Finally after the build finishes we re-extract the object
 # directory.
 #
 # Note that none of this is necessary if we're *not* in a docker-in-docker
 # scenario. If this script is run on a bare metal host then we share a bunch of
 # data directories to share as much data as possible. Note that we also use
 # `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all
 # read/written as the same user as the bare-metal user.
 if [ -f /.dockerenv ]; then
   docker create -v /checkout --name checkout alpine:3.4 /bin/true
   docker cp . checkout:/checkout
   args="$args --volumes-from checkout"
 else
   args="$args --volume $root_dir:/checkout:ro"
   args="$args --volume $objdir:/checkout/obj"
   args="$args --volume $HOME/.cargo:/cargo"
   args="$args --volume $HOME/rustsrc:$HOME/rustsrc"
   args="$args --volume /tmp/toolstate:/tmp/toolstate"

   id=$(id -u)
   if [[ "$id" != 0 && "$(docker -v)" =~ ^podman ]]; then
     # Rootless podman creates a separate user namespace, where an inner
     # LOCAL_USER_ID will map to a different subuid range on the host.
     # The "keep-id" mode maps the current UID directly into the container.
     args="$args --env NO_CHANGE_USER=1 --userns=keep-id"
   else
     args="$args --env LOCAL_USER_ID=$id"
   fi
 fi

 if [ "$dev" = "1" ]
 then
   # Interactive + TTY
   args="$args -it"
   command="/bin/bash"
 else
   command="/checkout/src/ci/run.sh"
 fi

 if [ "$CI" != "" ]; then
   # Get some needed information for $BASE_COMMIT
   #
   # This command gets the last merge commit which we'll use as base to list
   # deleted files since then.
   BASE_COMMIT="$(git log [email protected] -n 2 --pretty=format:%H | tail -n 1)"
 else
   BASE_COMMIT=""
 fi

 docker \
   run \
   --workdir /checkout/obj \
   --env SRC=/checkout \
   $args \
   --env CARGO_HOME=/cargo \
   --env DEPLOY \
   --env DEPLOY_ALT \
   --env CI \
   --env GITHUB_ACTIONS \
   --env GITHUB_REF \
   --env TOOLSTATE_REPO_ACCESS_TOKEN \
   --env TOOLSTATE_REPO \
   --env TOOLSTATE_PUBLISH \
   --env RUST_CI_OVERRIDE_RELEASE_CHANNEL \
   --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \
   --env BASE_COMMIT="$BASE_COMMIT" \
   --env DIST_TRY_BUILD \
   --env PR_CI_JOB \
   --env OBJDIR_ON_HOST="$objdir" \
   --init \
   --rm \
   rust-ci \
   $command

 if [ -f /.dockerenv ]; then
   rm -rf $objdir
   docker cp checkout:/checkout/obj $objdir
 fi
	#!/usr/bin/env bash

	set -e

	export MSYS_NO_PATHCONV=1

	script=`cd $(dirname $0) && pwd`/`basename $0`

	image=""
	dev=0

	while [[ $# -gt 0 ]]
	do
	case "$1" in
	--dev)
	dev=1
	;;
	*)
	if [ -n "$image" ]
	then
	echo "expected single argument for the image name"
	exit 1
	fi
	image="$1"
	;;
	esac
	shift
	done

	script_dir="`dirname $script`"
	docker_dir="${script_dir}/host-$(uname -m)"
	ci_dir="`dirname $script_dir`"
	src_dir="`dirname $ci_dir`"
	root_dir="`dirname $src_dir`"

	objdir=$root_dir/obj
	dist=$objdir/build/dist

	source "$ci_dir/shared.sh"

	CACHE_DOMAIN="${CACHE_DOMAIN:-ci-caches.rust-lang.org}"

	if [ -f "$docker_dir/$image/Dockerfile" ]; then
	if [ "$CI" != "" ]; then
	hash_key=/tmp/.docker-hash-key.txt
	rm -f "${hash_key}"
	echo $image >> $hash_key

	cat "$docker_dir/$image/Dockerfile" >> $hash_key
	# Look for all source files involves in the COPY command
	copied_files=/tmp/.docker-copied-files.txt
	rm -f "$copied_files"
	for i in $(sed -n -e '/^COPY --from=/! s/^COPY \(.\) .$/\1/p' \
	"$docker_dir/$image/Dockerfile"); do
	# List the file names
	find "$script_dir/$i" -type f >> $copied_files
	done
	# Sort the file names and cat the content into the hash key
	sort $copied_files \| xargs cat >> $hash_key

	# Include the architecture in the hash key, since our Linux CI does not
	# only run in x86_64 machines.
	uname -m >> $hash_key

	docker --version >> $hash_key

	# Include cache version. Can be used to manually bust the Docker cache.
	echo "2" >> $hash_key

	cksum=$(sha512sum $hash_key \| \
	awk '{print $1}')

	url="https://$CACHE_DOMAIN/docker/$cksum"

	echo "Attempting to download $url"
	rm -f /tmp/rustci_docker_cache
	set +e
	retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \
	-o /tmp/rustci_docker_cache "$url"

	docker_archive_hash=$(sha512sum /tmp/rustci_docker_cache \| awk '{print $1}')
	echo "Downloaded archive hash: ${docker_archive_hash}"

	echo "Loading images into docker"
	# docker load sometimes hangs in the CI, so time out after 10 minutes with TERM,
	# KILL after 12 minutes
	loaded_images=$(/usr/bin/timeout -k 720 600 docker load -i /tmp/rustci_docker_cache \
	\| sed 's/.* sha/sha/')
	set -e
	printf "Downloaded containers:\n$loaded_images\n"
	fi

	dockerfile="$docker_dir/$image/Dockerfile"
	if [ -x /usr/bin/cygpath ]; then
	context="`cygpath -w $script_dir`"
	dockerfile="`cygpath -w $dockerfile`"
	else
	context="$script_dir"
	fi
	echo "::group::Building docker image for $image"

	# As of August 2023, Github Actions have updated Docker to 23.X,
	# which uses the BuildKit by default. It currently throws aways all
	# intermediate layers, which breaks our usage of S3 layer caching.
	# Therefore we opt-in to the old build backend for now.
	export DOCKER_BUILDKIT=0
	retry docker \
	build \
	--rm \
	-t rust-ci \
	-f "$dockerfile" \
	"$context"
	echo "::endgroup::"

	if [ "$CI" != "" ]; then
	s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
	upload="aws s3 cp - $s3url"
	digest=$(docker inspect rust-ci --format '{{.Id}}')
	echo "Built container $digest"
	if ! grep -q "$digest" <(echo "$loaded_images"); then
	echo "Uploading finished image $digest to $url"
	set +e
	# Print image history for easier debugging of layer SHAs
	docker history rust-ci
	docker history -q rust-ci \| \
	grep -v missing \| \
	xargs docker save \| \
	gzip \| \
	$upload
	set -e
	else
	echo "Looks like docker image is the same as before, not uploading"
	fi
	# Record the container image for reuse, e.g. by rustup.rs builds
	info="$dist/image-$image.txt"
	mkdir -p "$dist"
	echo "$url" >"$info"
	echo "$digest" >>"$info"
	cat "$info"
	fi
	elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
	if isCI; then
	echo Cannot run disabled images on CI!
	exit 1
	fi
	# Transform changes the context of disabled Dockerfiles to match the enabled ones
	tar --transform 's#disabled/#./#' -C $script_dir -c . \| docker \
	build \
	--rm \
	-t rust-ci \
	-f "host-$(uname -m)/$image/Dockerfile" \
	-
	else
	echo Invalid image: $image

	# Check whether the image exists for other architectures
	for arch_dir in "${script_dir}"/host-*; do
	# Avoid checking non-directories and the current host architecture directory
	if ! [[ -d "${arch_dir}" ]]; then
	continue
	fi
	if [[ "${arch_dir}" = "${docker_dir}" ]]; then
	continue
	fi

	arch_name="$(basename "${arch_dir}" \| sed 's/^host-//')"
	if [[ -f "${arch_dir}/${image}/Dockerfile" ]]; then
	echo "Note: the image exists for the ${arch_name} host architecture"
	elif [[ -f "${arch_dir}/disabled/${image}/Dockerfile" ]]; then
	echo "Note: the disabled image exists for the ${arch_name} host architecture"
	else
	continue
	fi
	echo "Note: the current host architecture is $(uname -m)"
	done

	exit 1
	fi

	mkdir -p $HOME/.cargo
	mkdir -p $objdir/tmp
	mkdir -p $objdir/cores
	mkdir -p /tmp/toolstate

	args=
	if [ "$SCCACHE_BUCKET" != "" ]; then
	args="$args --env SCCACHE_BUCKET"
	args="$args --env SCCACHE_REGION"
	args="$args --env AWS_ACCESS_KEY_ID"
	args="$args --env AWS_SECRET_ACCESS_KEY"
	args="$args --env AWS_REGION"
	else
	mkdir -p $HOME/.cache/sccache
	args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache"
	fi

	# Run containers as privileged as it should give them access to some more
	# syscalls such as ptrace and whatnot. In the upgrade to LLVM 5.0 it was
	# discovered that the leak sanitizer apparently needs these syscalls nowadays so
	# we'll need `--privileged` for at least the `x86_64-gnu` builder, so this just
	# goes ahead and sets it for all builders.
	args="$args --privileged"

	# Things get a little weird if this script is already running in a docker
	# container. If we're already in a docker container then we assume it's set up
	# to do docker-in-docker where we have access to a working `docker` command.
	#
	# If this is the case (we check via the presence of `/.dockerenv`)
	# then we can't actually use the `--volume` argument. Typically we use
	# `--volume` to efficiently share the build and source directory between this
	# script and the container we're about to spawn. If we're inside docker already
	# though the `--volume` argument maps the host's folder to the container we're
	# about to spawn, when in fact we want the folder in this container itself. To
	# work around this we use a recipe cribbed from
	# https://circleci.com/docs/2.0/building-docker-images/#mounting-folders to
	# create a temporary container with a volume. We then copy the entire source
	# directory into this container, and then use that copy in the container we're
	# about to spawn. Finally after the build finishes we re-extract the object
	# directory.
	#
	# Note that none of this is necessary if we're not in a docker-in-docker
	# scenario. If this script is run on a bare metal host then we share a bunch of
	# data directories to share as much data as possible. Note that we also use
	# `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all
	# read/written as the same user as the bare-metal user.
	if [ -f /.dockerenv ]; then
	docker create -v /checkout --name checkout alpine:3.4 /bin/true
	docker cp . checkout:/checkout
	args="$args --volumes-from checkout"
	else
	args="$args --volume $root_dir:/checkout:ro"
	args="$args --volume $objdir:/checkout/obj"
	args="$args --volume $HOME/.cargo:/cargo"
	args="$args --volume $HOME/rustsrc:$HOME/rustsrc"
	args="$args --volume /tmp/toolstate:/tmp/toolstate"

	id=$(id -u)
	if [[ "$id" != 0 && "$(docker -v)" =~ ^podman ]]; then
	# Rootless podman creates a separate user namespace, where an inner
	# LOCAL_USER_ID will map to a different subuid range on the host.
	# The "keep-id" mode maps the current UID directly into the container.
	args="$args --env NO_CHANGE_USER=1 --userns=keep-id"
	else
	args="$args --env LOCAL_USER_ID=$id"
	fi
	fi

	if [ "$dev" = "1" ]
	then
	# Interactive + TTY
	args="$args -it"
	command="/bin/bash"
	else
	command="/checkout/src/ci/run.sh"
	fi

	if [ "$CI" != "" ]; then
	# Get some needed information for $BASE_COMMIT
	#
	# This command gets the last merge commit which we'll use as base to list
	# deleted files since then.
	BASE_COMMIT="$(git log [email protected] -n 2 --pretty=format:%H \| tail -n 1)"
	else
	BASE_COMMIT=""
	fi

	docker \
	run \
	--workdir /checkout/obj \
	--env SRC=/checkout \
	$args \
	--env CARGO_HOME=/cargo \
	--env DEPLOY \
	--env DEPLOY_ALT \
	--env CI \
	--env GITHUB_ACTIONS \
	--env GITHUB_REF \
	--env TOOLSTATE_REPO_ACCESS_TOKEN \
	--env TOOLSTATE_REPO \
	--env TOOLSTATE_PUBLISH \
	--env RUST_CI_OVERRIDE_RELEASE_CHANNEL \
	--env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \
	--env BASE_COMMIT="$BASE_COMMIT" \
	--env DIST_TRY_BUILD \
	--env PR_CI_JOB \
	--env OBJDIR_ON_HOST="$objdir" \
	--init \
	--rm \
	rust-ci \
	$command

	if [ -f /.dockerenv ]; then
	rm -rf $objdir
	docker cp checkout:/checkout/obj $objdir
	fi