🐛 Location-agnostic sign release

release.sh

@@ -111,6 +111,7 @@ export KO_DOCKER_REPO="gcr.io/knative-nightly"
 # Build stripped binary to reduce size
 export GOFLAGS="-ldflags=-s -ldflags=-w"
 export GITHUB_TOKEN=""
+readonly IMAGES_REFS_FILE="${IMAGES_REFS_FILE:-$(mktemp -d)/images_refs.txt}"
 
 # Convenience function to run the hub tool.
 # Parameters: $1..$n - arguments to hub.
@@ -313,64 +314,109 @@ function build_from_source() {
 }
 
 function get_images_in_yamls() {
-  rm -rf imagerefs.txt
+  rm -rf "$IMAGES_REFS_FILE"
   echo "Assembling a list of image refences to sign"
-  for file in $@; do
+  for file in "$@"; do
     [[ "${file##*.}" != "yaml" ]] && continue
     echo "Inspecting ${file}"
-    for image in $(grep -oh "\S*${KO_DOCKER_REPO}\S*" "${file}"); do
-      echo $image >> imagerefs.txt
-    done
+    while read -r image; do
+      echo "$image" >> "$IMAGES_REFS_FILE"
+    done < <(grep -oh "\S*${KO_DOCKER_REPO}\S*" "${file}")
+  done
+  if [[ -f "$IMAGES_REFS_FILE" ]]; then
+    sort -uo "$IMAGES_REFS_FILE" "$IMAGES_REFS_FILE" # Remove duplicate entries
+  fi
+}
+
+function find_checksums_file() {
+  for file in "$@"; do
+    if [[ "${file}" == *"checksums.txt" ]]; then
+      echo "${file}"
+      return 0
+    fi
   done
-  sort -uo imagerefs.txt imagerefs.txt # Remove duplicate entries
+  warning "cannot find checksums file"
 }
 
 # Build a release from source.
 function sign_release() {
-  get_images_in_yamls "${ARTIFACTS_TO_PUBLISH}"
   if (( ! IS_PROW )); then # This function can't be run by devs on their laptops
     return 0
   fi
+  get_images_in_yamls "${ARTIFACTS_TO_PUBLISH}"
+  local checksums_file
+  checksums_file="$(find_checksums_file "${ARTIFACTS_TO_PUBLISH}")"
+
+  if ! [[ -f "${checksums_file}" ]]; then
+    echo '>> No checksums file found, generating one'
+    checksums_file="$(mktemp -d)/checksums.txt"
+    for file in ${ARTIFACTS_TO_PUBLISH}; do
+      pushd "$(dirname "$file")" >/dev/null
+      sha256sum "$(basename "$file")" >> "${checksums_file}"
+      popd >/dev/null
+    done
+    ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} ${checksums_file}"
+  fi
 
   # Notarizing mac binaries needs to be done before cosign as it changes the checksum values
   # of the darwin binaries
  if [ -n "${APPLE_CODESIGN_KEY}" ] && [ -n "${APPLE_CODESIGN_PASSWORD_FILE}" ] && [ -n "${APPLE_NOTARY_API_KEY}" ]; then
     banner "Notarizing macOS Binaries for the release"
-    FILES=$(find -- * -type f -name "*darwin*")
-    for file in $FILES; do
-      rcodesign sign "${file}" --p12-file="${APPLE_CODESIGN_KEY}" \
-        --code-signature-flags=runtime \
-        --p12-password-file="${APPLE_CODESIGN_PASSWORD_FILE}"
-    done
-    zip files.zip ${FILES}
-    rcodesign notary-submit files.zip --api-key-path="${APPLE_NOTARY_API_KEY}" --wait
-    sha256sum ${ARTIFACTS_TO_PUBLISH//checksums.txt/} > checksums.txt
-    echo "🧮     Post Notarization Checksum:"
-    cat checksums.txt
+    local macos_artifacts
+    declare -a macos_artifacts=()
+    while read -r file; do
+      if echo "$file" | grep -q "darwin"; then
+        macos_artifacts+=("${file}")
+        rcodesign sign "${file}" --p12-file="${APPLE_CODESIGN_KEY}" \
+          --code-signature-flags=runtime \
+          --p12-password-file="${APPLE_CODESIGN_PASSWORD_FILE}"
+      fi
+    done < <(echo "${ARTIFACTS_TO_PUBLISH}" | tr ' ' '\n')
+    if [[ -z "${macos_artifacts[*]}" ]]; then
+      warning "No macOS binaries found, skipping notarization"
+    else
+      local zip_file
+      zip_file="$(mktemp -d)/files.zip"
+      zip "$zip_file" -@ < <(printf "%s\n"  "${macos_artifacts[@]}")
+      rcodesign notary-submit "$zip_file" --api-key-path="${APPLE_NOTARY_API_KEY}" --wait
+      true > "${checksums_file}" # Clear the checksums file
+      for file in ${ARTIFACTS_TO_PUBLISH}; do
+        if echo "$file" | grep -q "checksums.txt"; then
+          continue # Don't checksum the checksums file
+        fi
+        pushd "$(dirname "$file")" >/dev/null
+        sha256sum "$(basename "$file")" >> "${checksums_file}"
+        popd >/dev/null
+      done
+      echo "🧮     Post Notarization Checksum:"
+      cat "$checksums_file"
+    fi
   fi
 
   ID_TOKEN=$(gcloud auth print-identity-token --audiences=sigstore \
     --include-email \
     --impersonate-service-account="${SIGNING_IDENTITY}")
   echo "Signing Images with the identity ${SIGNING_IDENTITY}"
   ## Sign the images with cosign
-  if [[ -f "imagerefs.txt" ]]; then
-      COSIGN_EXPERIMENTAL=1 cosign sign $(cat imagerefs.txt) --recursive --identity-token="${ID_TOKEN}"
-      if  [ -n "${ATTEST_IMAGES:-}" ]; then # Temporary Feature Gate
-        provenance-generator --clone-log=/logs/clone.json \
-          --image-refs=imagerefs.txt --output=attestation.json
-        mkdir -p "${ARTIFACTS}"/attestation && cp attestation.json "${ARTIFACTS}"/attestation
-        COSIGN_EXPERIMENTAL=1 cosign attest $(cat imagerefs.txt) --recursive --identity-token="${ID_TOKEN}" \
-          --predicate=attestation.json --type=slsaprovenance
-      fi
+  if [[ -f "$IMAGES_REFS_FILE" ]]; then
+    COSIGN_EXPERIMENTAL=1 cosign sign $(cat "$IMAGES_REFS_FILE") \
+      --recursive --identity-token="${ID_TOKEN}"
+    if  [ -n "${ATTEST_IMAGES:-}" ]; then # Temporary Feature Gate
+      provenance-generator --clone-log=/logs/clone.json \
+        --image-refs="$IMAGES_REFS_FILE" --output=attestation.json
+      mkdir -p "${ARTIFACTS}"/attestation && cp attestation.json "${ARTIFACTS}"/attestation
+      COSIGN_EXPERIMENTAL=1 cosign attest $(cat "$IMAGES_REFS_FILE") \
+        --recursive --identity-token="${ID_TOKEN}" \
+        --predicate=attestation.json --type=slsaprovenance
+    fi
   fi
 
-  ## Check if there is checksums.txt file. If so, sign the checksum file
-  if [[ -f "checksums.txt" ]]; then
-      echo "Signing Images with the identity ${SIGNING_IDENTITY}"
-      COSIGN_EXPERIMENTAL=1 cosign sign-blob checksums.txt --output-signature=checksums.txt.sig --output-certificate=checksums.txt.pem --identity-token="${ID_TOKEN}"
-      ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} checksums.txt.sig checksums.txt.pem"
-  fi
+  echo "Signing checksums with the identity ${SIGNING_IDENTITY}"
+  COSIGN_EXPERIMENTAL=1 cosign sign-blob "$checksums_file" \
+    --output-signature="${checksums_file}.sig" \
+    --output-certificate="${checksums_file}.pem" \
+    --identity-token="${ID_TOKEN}"
+  ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} ${checksums_file}.sig ${checksums_file}.pem"
 }
 
 # Copy tagged images from the nightly GCR to the release GCR, tagging them 'latest'.

test/unit/release_test.go

@@ -2,12 +2,64 @@ package unit_test
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
+func TestRelease(t *testing.T) {
+	t.Parallel()
+
+	outChecks := []check{
+		contains("Signing Images with the identity signer@knative-releases.test"),
+		contains("Signing checksums with the identity signer@knative-releases.test"),
+		contains("Notarizing macOS Binaries for the release"),
+	}
+	checksumsContent := `🧮     Post Notarization Checksum:
+4d410c6611b89b21215e06046dc8104aa668c8e93a5b73062e45bd43c6c422cc  foo-linux-amd64
+6fedd2d0b79cbd3faf11f159f6b229707e191a5bcc5f727fd33b916d517c8ed4  foo-linux-arm64
+58eaa00b44cb836d09f009791bdb2c521afc18f7a2dac80422a6204774d6a677  foo-linux-ppc64le
+7b33c5e58372290a7addc5e9b95a1fef33bb1ce38660dd4fdc65b9862e466a59  foo-linux-s390x
+9ee0670b6715542ef64a336ae68342fde32d3045273dcfe67d97c22f72f4c039  foo-darwin-amd64
+74da512cfed7a90713a7161f34a2339fe2e9c9cec8bd3cb30566c464bf2c18f1  foo-darwin-arm64
+73517e997b68696b1a6be4957519b800e26c9bc44c1b7f46fe90be0834d1af07  foo-windows-amd64.exe
+9ac630646ca5b77fbf716f9a780d33f26357bbd8b242c14e0863cdde72aacbf0  foo.yaml
+`
+	tcs := []testCase{{
+		name:    "build_from_source",
+		retcode: retcode(0),
+		stderr:  warned("cannot find checksums file"),
+		stdout:  outChecks,
+	}, {
+		name: "build_from_source (with_checksums)",
+		commands: []string{
+			`export CALCULATE_CHECKSUMS=1`,
+			`build_from_source`,
+		},
+		stdout: append(outChecks, contains(checksumsContent)),
+	}}
+	for _, tc := range tcs {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			tmp := t.TempDir()
+			sc := testReleaseShellScript(
+				envs(map[string]string{
+					"BUILD_DIR":                    tmp,
+					"APPLE_CODESIGN_KEY":           randomFile(t, tmp, "codesign.key"),
+					"APPLE_CODESIGN_PASSWORD_FILE": randomFile(t, tmp, "codesign.pass"),
+					"APPLE_NOTARY_API_KEY":         randomFile(t, tmp, "notary.key"),
+					"SIGNING_IDENTITY":             "signer@knative-releases.test",
+				}),
+				loadFile("fake-build-release.bash"),
+			)
+			tc.test(sc)(t)
+		})
+	}
+}
+
 func TestReleaseHelperFunctions(t *testing.T) {
 	t.Parallel()
 	sc := testReleaseShellScript()
@@ -256,9 +308,22 @@ func TestReleaseFlagParsingDefaults(t *testing.T) {
 	}
 }
 
-func testReleaseShellScript() shellScript {
-	return newShellScript(
+func testReleaseShellScript(scrps ...scriptlet) shellScript {
+	aargs := make([]scriptlet, 0, len(scrps)+3)
+	aargs = append(aargs,
 		fakeProwJob(),
 		loadFile("source-release.bash"),
+		loadFile("fake-presubmit-tests.bash"),
 	)
+	aargs = append(aargs, scrps...)
+	return newShellScript(aargs...)
+}
+
+func randomFile(tb testing.TB, tmpdir string, filename string) string {
+	r := randString(24)
+	fp := path.Join(tmpdir, filename)
+	if err := ioutil.WriteFile(fp, []byte(r), 0o600); err != nil {
+		tb.Fatal(err)
+	}
+	return fp
 }

test/unit/scripts/fake-build-release.bash

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -Eeuo pipefail
+
+build_dir="${BUILD_DIR:-$(mktemp -d)}"
+export ARTIFACTS_TO_PUBLISH
+
+function build_release() {
+  local artifact_names
+  declare -a artifact_names
+  artifact_names=(
+    foo-linux-amd64
+    foo-linux-arm64
+    foo-linux-ppc64le
+    foo-linux-s390x
+    foo-darwin-amd64
+    foo-darwin-arm64
+    foo-windows-amd64.exe
+    foo.yaml
+  )
+  for artifact_name in "${artifact_names[@]}"; do
+    hashCode "$artifact_name" > "${build_dir}/${artifact_name}"
+    echo "${build_dir}/${artifact_name}" >> "${build_dir}/artifacts.list"
+  done
+  ARTIFACTS_TO_PUBLISH="$(tr '\r\n' ' ' < "${build_dir}/artifacts.list")"
+  if [[ -n "${CALCULATE_CHECKSUMS:-}" ]]; then
+    calculate_checksums
+  fi
+}
+
+function calculate_checksums {
+  local checksums file
+  checksums="${build_dir}/checksums.txt"
+  rm -vf "${checksums}"
+  while read -r file; do
+    pushd "$(dirname "$file")" >/dev/null
+    sha256sum "$(basename "$file")" >> "${checksums}"
+    popd >/dev/null
+  done < "${build_dir}/artifacts.list"
+  ARTIFACTS_TO_PUBLISH="${ARTIFACTS_TO_PUBLISH} ${checksums}"
+}

test/unit/scripts/fake-presubmit-tests.bash

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_trap teardown_fake_presubmit EXIT
+
+VALIDATION_TESTS="$(mktemp -d)/presubmit-tests.sh"
+
+cat <<EOF > "${VALIDATION_TESTS}"
+#!/usr/bin/env bash
+
+echo ">> Running fake presubmit tests"
+echo "UNIT TESTS PASSED"
+echo "INTEGRATION TESTS PASSED"
+EOF
+
+chmod +x "${VALIDATION_TESTS}"
+
+function teardown_fake_presubmit() {
+  rm -rf "${VALIDATION_TESTS}"
+}

test/unit/sharedlib_test.go

@@ -30,6 +30,11 @@ func aborted(msg string) []check {
 	return equal(makeBanner('*', fmsg))
 }
 
+func warned(msg string) []check {
+	fmsg := fmt.Sprintf("WARN: %s", msg)
+	return equal(makeBanner('!', fmsg))
+}
+
 func makeBanner(ch rune, msg string) string {
 	const span = 4
 	border := strings.Repeat(string(ch), len(msg)+span*2+2)
@@ -169,6 +174,14 @@ func loadFile(names ...string) scriptlet {
 	})
 }
 
+func envs(envs map[string]string) scriptlet {
+	instr := make([]string, 0, len(envs))
+	for k, v := range envs {
+		instr = append(instr, fmt.Sprintf(`export %s="%s"`, k, v))
+	}
+	return instructions(instr...)
+}
+
 func instructions(inst ...string) scriptlet {
 	return fnScriptlet(func(t TestingT) string {
 		return strings.Join(inst, "\n")
@@ -282,11 +295,16 @@ func mockKubectl(responses ...response) scriptlet {
 func fakeProwJob() scriptlet {
 	return union(
 		loadFile("fake-prow-job.bash"),
-		mockBinary("gcloud"),
+		mockBinary("gcloud", response{
+			startsWith{"auth print-identity-token"},
+			simply("F4KE-T0K3N-3B49"),
+		}),
 		mockBinary("java"),
 		mockBinary("mvn"),
 		mockBinary("ko"),
 		mockBinary("cosign"),
+		mockBinary("rcodesign"),
+		mockBinary("gsutil"),
 	)
 }