Add explicit "external-pins", take two

This is a second attempt at 98575e0:

> The goal of this directory is two-fold: to list the explicitly allowable external dependencies and to pin those external dependencies to a specific content-addressable digest such that we can update them in the same way we do everything else (and then trigger rebuilds properly based on them, too).

See `.external-pins/list.sh` for a useful summary of the assumptions that went into the naming scheme.

Author: tianon

.external-pins/list.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+dir="$(dirname "$BASH_SOURCE")"
+
+find "$dir" -mindepth 2 -type f -printf '%P\n' | sed -e 's/___/:/' | sort
+
+# assumptions which make the "___" -> ":" conversion ~safe (examples referencing "example.com/foo/bar:baz"):
+#
+#   1. we *always* specify a tag ("baz")
+#   2. the domain ("example.com") cannot contain underscores
+#   3. we do not pin to any registry with a non-443 port ("example.com:8443")
+#   4. the repository ("foo/bar") can only contain singular or double underscores (never triple underscore), and only between alphanumerics (thus never right up next to ":")
+#   5. we do *not* use the "g" regex modifier in our sed, which means only the first instance of triple underscore is replaced (in pure Bash, that's "${img/:/___}" or "${img/___/:}" depending on the conversion direction)
+#
+# see https://github.com/distribution/distribution/blob/411d6bcfd2580d7ebe6e346359fa16aceec109d5/reference/regexp.go
+# (see also https://github.com/docker-library/perl-bashbrew/blob/6685582f7889ef4806f0544b93f10640c7608b1a/lib/Bashbrew/RemoteImageRef.pm#L9-L26 for a condensed version)
+#
+# see https://github.com/docker-library/official-images/issues/13608 for why we can't just use ":" as-is (even though Linux, macOS, and even Windows via MSYS / WSL2 don't have any issues with it)

.external-pins/mcr.microsoft.com/windows/nanoserver___1809

@@ -0,0 +1 @@
+sha256:5c9da3bfe1524eab610dfd41c79a0805c080f8307d810d62bba106b5df587698

.external-pins/mcr.microsoft.com/windows/nanoserver___ltsc2022

@@ -0,0 +1 @@
+sha256:f1125cf932664b039a55afb666bdca7dc28d453a88430fe47be763e0270638b6

.external-pins/mcr.microsoft.com/windows/servercore___1809

@@ -0,0 +1 @@
+sha256:cbb8b0a709b4e0868cd2e30b1485358197b1021bb0dd4261e36b3af3ca48fd0b

.external-pins/mcr.microsoft.com/windows/servercore___ltsc2022

@@ -0,0 +1 @@
+sha256:8d862016bcfad6040cb9ae0f21b024fe71b058ddb42a077c32a24bcb5fc04628

.external-pins/redhat/ubi9-minimal___latest

@@ -0,0 +1 @@
+sha256:3685c58885771d2ea14608caeab6a4c3949b973588c29ce91f462b486ee1be25

.external-pins/update.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+dir="$(dirname "$BASH_SOURCE")"
+
+if [ "$#" -eq 0 ]; then
+	images="$("$dir/list.sh")"
+	set -- $images
+fi
+
+for img; do
+	echo -n "$img -> "
+
+	if [[ "$img" != *:* ]]; then
+		echo >&2 "error: '$img' does not contain ':' -- this violates our assumptions! (did you mean '$img:latest' ?)"
+		exit 1
+	fi
+
+	digest="$(bashbrew remote arches --json "$img" | jq -r '.desc.digest')"
+
+	imgFile="$dir/${img/:/___}" # see ".external-pins/list.sh"
+	imgDir="$(dirname "$imgFile")"
+	mkdir -p "$imgDir"
+	echo "$digest" | tee "$imgFile"
+done

.github/workflows/.bashbrew/action.yml

@@ -10,8 +10,8 @@ runs:
 
     # these two version numbers are intentionally as close together as I could possibly get them because no matter what I tried, GitHub will not allow me to DRY them (can't have any useful variables in `uses:` and can't even have YAML references to steal it in `env:` or something)
     - shell: 'bash -Eeuo pipefail -x {0}'
-      run:    echo BASHBREW_VERSION=v0.1.5 >> "$GITHUB_ENV"
-    - uses: docker-library/[email protected].5
+      run:    echo BASHBREW_VERSION=v0.1.6 >> "$GITHUB_ENV"
+    - uses: docker-library/[email protected].6
       if: inputs.build == 'host'
 
     - run: docker build --pull --tag oisupport/bashbrew:base "https://github.com/docker-library/bashbrew.git#$BASHBREW_VERSION"

README.md

@@ -119,11 +119,7 @@ When taking over an existing repository, please ensure that the entire Git histo
 
 Rebuilding the same `Dockerfile` should result in the same version of the image being packaged, even if the second build happens several versions later, or the build should fail outright, such that an inadvertent rebuild of a `Dockerfile` tagged as `0.1.0` doesn't end up containing `0.2.3`. For example, if using `apt` to install the main program for the image, be sure to pin it to a specific version (ex: `... apt-get install -y my-package=0.1.0 ...`). For dependent packages installed by `apt` there is not usually a need to pin them to a version.
 
-No official images can be derived from, or depend on, non-official images with the following notable exceptions:
-
--	[`FROM scratch`](https://hub.docker.com/_/scratch/)
--	[`FROM mcr.microsoft.com/windows/servercore`](https://hub.docker.com/r/microsoft/windowsservercore/)
--	[`FROM mcr.microsoft.com/windows/nanoserver`](https://hub.docker.com/r/microsoft/nanoserver/)
+No official images can be derived from, or depend on, non-official images (allowing the non-image [`scratch`](https://hub.docker.com/_/scratch/) and the intentionally limited exceptions pinned in [`.external-pins`](.external-pins) -- see also [`.external-pins/list.sh`](.external-pins/list.sh)).
 
 #### Consistency
 

naughty-from.sh

@@ -13,49 +13,67 @@ if [ "$#" -eq 0 ]; then
 	set -- '--all'
 fi
 
+externalPinsDir="$(dirname "$BASH_SOURCE")/.external-pins"
+declare -A externalPinsArchesCache=(
+	#[img:tag]='["arch","arch",...]' # (json array of strings)
+)
 _is_naughty() {
 	local from="$1"; shift
 
-	case "$BASHBREW_ARCH=$from" in
-		# a few images that no longer exist (and are thus not permissible)
-		# https://techcommunity.microsoft.com/t5/Containers/Removing-the-latest-Tag-An-Update-on-MCR/ba-p/393045
-		*=mcr.microsoft.com/windows/*:latest) return 0 ;;
-
-
-		# https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle
-		#*=mcr.microsoft.com/windows/*:ltsc2022) return 0 ;; # "10/13/2026"
-		#*=mcr.microsoft.com/windows/*:20H2*)    return 0 ;; # "05/10/2022" *technically*, but its use is discouraged here given the existence of ltsc2022
-		*=mcr.microsoft.com/windows/*:2004*)    return 0 ;; # "12/14/2021"
-		*=mcr.microsoft.com/windows/*:1909*)    return 0 ;; # "05/11/2021"
-		*=mcr.microsoft.com/windows/*:1903*)    return 0 ;; # "12/08/2020"
-		#*=mcr.microsoft.com/windows/*:1809*)    return 0 ;; # "01/09/2024"
-		*=mcr.microsoft.com/windows/*:1803*)    return 0 ;; # "11/12/2019"
-		*=mcr.microsoft.com/windows/*:1709*)    return 0 ;; # "04/09/2019"
-		*=mcr.microsoft.com/windows/*:ltsc2016) return 0 ;; # "01/11/2022"
-		*=mcr.microsoft.com/windows/*:sac2016)  return 0 ;; # "10/09/2018"
-		*=mcr.microsoft.com/windows/*:1607*)    return 0 ;; # "10/09/2018"
-
-		# a few explicitly permissible exceptions to Santa's naughty list
-		*=scratch \
-		| amd64=docker.elastic.co/elasticsearch/elasticsearch:* \
-		| amd64=docker.elastic.co/kibana/kibana:* \
-		| amd64=docker.elastic.co/logstash/logstash:* \
-		| arm64v8=docker.elastic.co/elasticsearch/elasticsearch:* \
-		| arm64v8=docker.elastic.co/kibana/kibana:* \
-		| arm64v8=docker.elastic.co/logstash/logstash:* \
-		| windows-*=mcr.microsoft.com/windows/nanoserver:* \
-		| windows-*=mcr.microsoft.com/windows/servercore:* \
-		) return 1 ;;
-
-		# "x/y" and not an approved exception
-		*/*) return 0 ;;
+	case "$from" in
+		# "scratch" isn't a real image and is always permissible (on non-Windows)
+		scratch)
+			case "$BASHBREW_ARCH" in
+				windows-*) return 0 ;; # can't use "FROM scratch" on Windows
+				*)         return 1 ;; # can use "FROM scratch" everywhere else
+			esac
+			;;
+
+		# https://github.com/docker-library/official-images/pull/4916#issuecomment-427437270
+		  docker.elastic.co/elasticsearch/elasticsearch:*@sha256:* \
+		| docker.elastic.co/kibana/kibana:*@sha256:* \
+		| docker.elastic.co/logstash/logstash:*@sha256:* \
+		) ;; # *technically* we should only whitelist these for "elasticsearch", "kibana", and "logstash" respectively, but the chances of other folks trying to use them in their images (*and* doing so without us noticing) seems low
+
+		*/*)
+			# must be external, let's check our pins for acceptability
+			local externalPinFile="$externalPinsDir/${from/:/___}" # see ".external-pins/list.sh"
+			if [ -s "$externalPinFile" ]; then
+				local digest
+				digest="$(< "$externalPinFile")"
+				from+="@$digest"
+			else
+				# not pinned, must not be acceptable
+				return 0
+			fi
+			;;
 	esac
 
-	# must be some other official image AND support our current architecture
-	local archSupported
-	if archSupported="$(bashbrew cat --format '{{ .TagEntry.HasArchitecture arch | ternary arch "" }}' "$from")" && [ -n "$archSupported" ]; then
-		return 1
-	fi
+	case "$from" in
+		*/*@sha256:*)
+			if [ -z "${externalPinsArchesCache["$from"]:-}" ]; then
+				local remoteArches
+				if remoteArches="$(bashbrew remote arches --json "$from" | jq -c '.arches | keys')"; then
+					externalPinsArchesCache["$from"]="$remoteArches"
+				else
+					echo >&2 "warning: failed to query supported architectures of '$from'"
+					externalPinsArchesCache["$from"]='[]'
+				fi
+			fi
+			if jq <<<"${externalPinsArchesCache["$from"]}" -e 'index(env.BASHBREW_ARCH)' > /dev/null; then
+				# hooray, a supported architecture!
+				return 1
+			fi
+			;;
+
+		*)
+			# must be some other official image AND support our current architecture
+			local archSupported
+			if archSupported="$(bashbrew cat --format '{{ .TagEntry.HasArchitecture arch | ternary arch "" }}' "$from")" && [ -n "$archSupported" ]; then
+				return 1
+			fi
+			;;
+	esac
 
 	return 0
 }