Merge pull request #33810 from thaJeztah/17.03-update-frozen-image-script17.03.x

[17.03.x] update frozen image script

1 files changed, 226 insertions, 40 deletions

diff --git a/contrib/download-frozen-image-v2.sh b/contrib/download-frozen-image-v2.sh
index 111e3fa2ba..e6dfa8962a 100755
--- a/contrib/download-frozen-image-v2.sh
+++ b/contrib/download-frozen-image-v2.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eo pipefail
 
 # hello-world                      latest              ef872312fe1b        3 months ago        910 B
 # hello-world                      latest              ef872312fe1bbc5e05aae626791a47ee9b032efa8f3bda39cc0be7b56bfe59b9   3 months ago        910 B
@@ -11,6 +11,10 @@ if ! command -v curl &> /dev/null; then
 	echo >&2 'error: "curl" not found!'
 	exit 1
 fi
+if ! command -v jq &> /dev/null; then
+	echo >&2 'error: "jq" not found!'
+	exit 1
+fi
 
 usage() {
 	echo "usage: $0 dir image[:tag][@digest] ..."
@@ -27,8 +31,55 @@ mkdir -p "$dir"
 # hacky workarounds for Bash 3 support (no associative arrays)
 images=()
 rm -f "$dir"/tags-*.tmp
+manifestJsonEntries=()
+doNotGenerateManifestJson=
 # repositories[busybox]='"latest": "...", "ubuntu-14.04": "..."'
 
+# bash v4 on Windows CI requires CRLF separator
+newlineIFS=$'\n'
+if [ "$(go env GOHOSTOS)" = 'windows' ]; then
+	major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1)
+	if [ "$major" -ge 4 ]; then
+		newlineIFS=$'\r\n'
+	fi
+fi
+
+registryBase='https://registry-1.docker.io'
+authBase='https://auth.docker.io'
+authService='registry.docker.io'
+
+# https://github.com/moby/moby/issues/33700
+fetch_blob() {
+	local token="$1"; shift
+	local image="$1"; shift
+	local digest="$1"; shift
+	local targetFile="$1"; shift
+	local curlArgs=( "$@" )
+
+	local curlHeaders="$(
+		curl -S "${curlArgs[@]}" \
+			-H "Authorization: Bearer $token" \
+			"$registryBase/v2/$image/blobs/$digest" \
+			-o "$targetFile" \
+			-D-
+	)"
+	curlHeaders="$(echo "$curlHeaders" | tr -d '\r')"
+	if [ "$(echo "$curlHeaders" | awk 'NR == 1 { print $2; exit }')" != '200' ]; then
+		rm -f "$targetFile"
+
+		local blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')"
+		if [ -z "$blobRedirect" ]; then
+			echo >&2 "error: failed fetching '$image' blob '$digest'"
+			echo "$curlHeaders" | head -1 >&2
+			return 1
+		fi
+
+		curl -fSL "${curlArgs[@]}" \
+			"$blobRedirect" \
+			-o "$targetFile"
+	fi
+}
+
 while [ $# -gt 0 ]; do
 	imageTag="$1"
 	shift
@@ -44,30 +95,183 @@ while [ $# -gt 0 ]; do
 
 	imageFile="${image//\//_}" # "/" can't be in filenames :)
 
-	token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)"
+	token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
 
-	manifestJson="$(curl -sSL -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/manifests/$digest")"
+	manifestJson="$(
+		curl -fsSL \
+			-H "Authorization: Bearer $token" \
+			-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
+			-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
+			"$registryBase/v2/$image/manifests/$digest"
+	)"
 	if [ "${manifestJson:0:1}" != '{' ]; then
 		echo >&2 "error: /v2/$image/manifests/$digest returned something unexpected:"
 		echo >&2 "  $manifestJson"
 		exit 1
 	fi
 
-	layersFs=$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')
+	imageIdentifier="$image:$tag@$digest"
 
-	IFS=$'\n'
-	# bash v4 on Windows CI requires CRLF separator
-	if [ "$(go env GOHOSTOS)" = 'windows' ]; then
-		major=$(echo ${BASH_VERSION%%[^0.9]} | cut -d. -f1)
-		if [ "$major" -ge 4 ]; then
-			IFS=$'\r\n'
-		fi
-	fi
-	layers=( ${layersFs} )
-	unset IFS
+	schemaVersion="$(echo "$manifestJson" | jq --raw-output '.schemaVersion')"
+	case "$schemaVersion" in
+		2)
+			mediaType="$(echo "$manifestJson" | jq --raw-output '.mediaType')"
+
+			case "$mediaType" in
+				application/vnd.docker.distribution.manifest.v2+json)
+					configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
+					imageId="${configDigest#*:}" # strip off "sha256:"
+
+					configFile="$imageId.json"
+					fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
+
+					layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
+					IFS="$newlineIFS"
+					layers=( $layersFs )
+					unset IFS
+
+					echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
+					layerId=
+					layerFiles=()
+					for i in "${!layers[@]}"; do
+						layerMeta="${layers[$i]}"
+
+						layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
+						layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"
+
+						# save the previous layer's ID
+						parentId="$layerId"
+						# create a new fake layer ID based on this layer's digest and the previous layer's fake ID
+						layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
+						# this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)
+
+						mkdir -p "$dir/$layerId"
+						echo '1.0' > "$dir/$layerId/VERSION"
+
+						if [ ! -s "$dir/$layerId/json" ]; then
+							parentJson="$(printf ', parent: "%s"' "$parentId")"
+							addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
+							# this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
+							jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
+								{
+									"created": "0001-01-01T00:00:00Z",
+									"container_config": {
+										"Hostname": "",
+										"Domainname": "",
+										"User": "",
+										"AttachStdin": false,
+										"AttachStdout": false,
+										"AttachStderr": false,
+										"Tty": false,
+										"OpenStdin": false,
+										"StdinOnce": false,
+										"Env": null,
+										"Cmd": null,
+										"Image": "",
+										"Volumes": null,
+										"WorkingDir": "",
+										"Entrypoint": null,
+										"OnBuild": null,
+										"Labels": null
+									}
+								}
+							EOJSON
+						fi
 
-	history=$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')
-	imageId=$(echo "$history" | jq --raw-output .[0] | jq --raw-output .id)
+						case "$layerMediaType" in
+							application/vnd.docker.image.rootfs.diff.tar.gzip)
+								layerTar="$layerId/layer.tar"
+								layerFiles=( "${layerFiles[@]}" "$layerTar" )
+								# TODO figure out why "-C -" doesn't work here
+								# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
+								# "HTTP/1.1 416 Requested Range Not Satisfiable"
+								if [ -f "$dir/$layerTar" ]; then
+									# TODO hackpatch for no -C support :'(
+									echo "skipping existing ${layerId:0:12}"
+									continue
+								fi
+								token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
+								fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
+								;;
+
+							*)
+								echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
+								exit 1
+								;;
+						esac
+					done
+
+					# change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
+					imageId="$layerId"
+
+					# munge the top layer image manifest to have the appropriate image configuration for older daemons
+					imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
+					jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"
+
+					manifestJsonEntry="$(
+						echo '{}' | jq --raw-output '. + {
+							Config: "'"$configFile"'",
+							RepoTags: ["'"${image#library\/}:$tag"'"],
+							Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
+						}'
+					)"
+					manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
+					;;
+
+				*)
+					echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'"
+					exit 1
+					;;
+			esac
+			;;
+
+		1)
+			if [ -z "$doNotGenerateManifestJson" ]; then
+				echo >&2 "warning: '$imageIdentifier' uses schemaVersion '$schemaVersion'"
+				echo >&2 "  this script cannot (currently) recreate the 'image config' to put in a 'manifest.json' (thus any schemaVersion 2+ images will be imported in the old way, and their 'docker history' will suffer)"
+				echo >&2
+				doNotGenerateManifestJson=1
+			fi
+
+			layersFs="$(echo "$manifestJson" | jq --raw-output '.fsLayers | .[] | .blobSum')"
+			IFS="$newlineIFS"
+			layers=( $layersFs )
+			unset IFS
+
+			history="$(echo "$manifestJson" | jq '.history | [.[] | .v1Compatibility]')"
+			imageId="$(echo "$history" | jq --raw-output '.[0]' | jq --raw-output '.id')"
+
+			echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
+			for i in "${!layers[@]}"; do
+				imageJson="$(echo "$history" | jq --raw-output ".[${i}]")"
+				layerId="$(echo "$imageJson" | jq --raw-output '.id')"
+				imageLayer="${layers[$i]}"
+
+				mkdir -p "$dir/$layerId"
+				echo '1.0' > "$dir/$layerId/VERSION"
+
+				echo "$imageJson" > "$dir/$layerId/json"
+
+				# TODO figure out why "-C -" doesn't work here
+				# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
+				# "HTTP/1.1 416 Requested Range Not Satisfiable"
+				if [ -f "$dir/$layerId/layer.tar" ]; then
+					# TODO hackpatch for no -C support :'(
+					echo "skipping existing ${layerId:0:12}"
+					continue
+				fi
+				token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
+				fetch_blob "$token" "$image" "$imageLayer" "$dir/$layerId/layer.tar" --progress
+			done
+			;;
+
+		*)
+			echo >&2 "error: unknown manifest schemaVersion ($imageIdentifier): '$schemaVersion'"
+			exit 1
+			;;
+	esac
+
+	echo
 
 	if [ -s "$dir/tags-$imageFile.tmp" ]; then
 		echo -n ', ' >> "$dir/tags-$imageFile.tmp"
@@ -75,30 +279,6 @@ while [ $# -gt 0 ]; do
 		images=( "${images[@]}" "$image" )
 	fi
 	echo -n '"'"$tag"'": "'"$imageId"'"' >> "$dir/tags-$imageFile.tmp"
-
-	echo "Downloading '${image}:${tag}@${digest}' (${#layers[@]} layers)..."
-	for i in "${!layers[@]}"; do
-		imageJson=$(echo "$history" | jq --raw-output .[${i}])
-		imageId=$(echo "$imageJson" | jq --raw-output .id)
-		imageLayer=${layers[$i]}
-
-		mkdir -p "$dir/$imageId"
-		echo '1.0' > "$dir/$imageId/VERSION"
-
-		echo "$imageJson" > "$dir/$imageId/json"
-
-		# TODO figure out why "-C -" doesn't work here
-		# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
-		# "HTTP/1.1 416 Requested Range Not Satisfiable"
-		if [ -f "$dir/$imageId/layer.tar" ]; then
-			# TODO hackpatch for no -C support :'(
-			echo "skipping existing ${imageId:0:12}"
-			continue
-		fi
-		token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)"
-		curl -SL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$imageId/layer.tar" # -C -
-	done
-	echo
 done
 
 echo -n '{' > "$dir/repositories"
@@ -116,6 +296,12 @@ echo -n $'\n}\n' >> "$dir/repositories"
 
 rm -f "$dir"/tags-*.tmp
 
+if [ -z "$doNotGenerateManifestJson" ] && [ "${#manifestJsonEntries[@]}" -gt 0 ]; then
+	echo '[]' | jq --raw-output ".$(for entry in "${manifestJsonEntries[@]}"; do echo " + [ $entry ]"; done)" > "$dir/manifest.json"
+else
+	rm -f "$dir/manifest.json"
+fi
+
 echo "Download of images into '$dir' complete."
 echo "Use something like the following to load the result into a Docker daemon:"
 echo "  tar -cC '$dir' . | docker load"