New script tarball_multi_releases.sh to combine releases

tarball_multi_releases.bash

@@ -0,0 +1,241 @@
+#!/usr/bin/env bash
+
+set -e
+
+usage()
+{
+    cat <<EOF
+
+Combine SOF firmware files from different versions/directories into a
+single lib/firmware/intel/sof* release tarball.
+
+- Creates a complete archive from git (to ignore uncommitted binaries)
+- Removes all but the given versions from it
+- Merges files from the given versions _in the order given on the
+  command line_. In case of conflict last on the command line wins.
+- Records which file came from which version in manifest.txt
+
+The archive is named after the version number in the last argument.
+
+Example:
+        $0 -g v2.3 v2.0.x/tools-v2.0 v2.1.x/sof-tplg-v2.1.1 v2.2.x/sof-v2.2.1 v2.3.x/sof-v2.3
+
+This example will create a sof-bin-v2.3.tar.gz archive; its files will
+extract in directories sof-v2.3/, tools-v2.3/ and sof-tplg-v2.3/
+
+Synopsis:
+        $0 [ -g optional_git_ref ] dir1 dir2 dir3 ...
+
+The git reference is optional because most files are immutable. Some
+tests need -g.
+
+EOF
+    exit 1
+}
+
+if [ "$(uname)" = 'Darwin' ]; then
+    find() { # for -xtype
+        gfind "$@"  # brew install findutils
+    }
+fi
+
+main()
+{
+    local gittop; gittop="$(git rev-parse --show-toplevel)"
+
+    parse_args "$@"
+
+    printf "Archiving these directories in this order, last one wins: %s\n" "${TOP_DIRS[*]}"
+
+    local last_dir last_ver
+    # This is a single-element array slice.
+    # shellcheck disable=SC2124
+    last_dir="${TOP_DIRS[@]: -1}"
+    last_ver=$(parse_version_suffix "$last_dir")
+    local archive_name=sof-bin-"$last_ver"
+
+    if test -e "$archive_name"; then
+        die "%s already exists\n" "$archive_name"
+    fi
+
+    ( local _pwd; _pwd=$(pwd)
+      set -x
+      # Start with a clean git archive and the version number in the
+      # top-level directory.
+      cd "${gittop}" # git archive is painful like that
+      git archive -o "${_pwd}"/_.tar --prefix="$archive_name"/ "$GIT_REF"
+    )
+    tar xf _.tar; rm _.tar
+
+    # Select what we want in the tarball
+    rm -rf _selected_versions;  mkdir _selected_versions
+    ( set -ex
+      local version_dir
+      for version_dir in "${TOP_DIRS[@]}"; do
+          mv "$archive_name"/"$version_dir"  _selected_versions/
+      done
+    )
+    # Pick up ancillary files
+    ( set -e
+      local _pwd; _pwd=$(pwd)
+      cd "${archive_name:?}"
+      rm tarball*sh
+      rm -f README-before-1.7.md
+      set -x
+      mv install.sh README* LICENCE* Notice* "${_pwd}"/_selected_versions/
+    )
+
+    # Delete everything else
+    rm -r "${archive_name:?}"/* "${archive_name:?}"/.github
+
+    # Move the selected things back in place
+    (set -x; mv _selected_versions/* "$archive_name"/ )
+    rmdir _selected_versions
+
+    check_symlinks "$archive_name" ||
+        die "Found some broken symbolic links before combining\n"
+
+    # Now "install" versions in the given order and on top of each
+    # other: last one wins. Record the version of each file for the
+    # manifest.
+    local p ver dir radix
+    for p in "${TOP_DIRS[@]}"; do
+        topdir=$(basename "$p")
+        ver=$(parse_version_suffix "$p")
+        radix=${topdir%-"$ver"}
+        record_files_version "${archive_name}"  "$topdir"  "${ver}"
+        # copy + delete ~= move
+        ( set -x
+          rsync -a --checksum  "${archive_name}"/"$topdir"/  "${archive_name}/${radix}"-WIP/
+        )
+        rm -rf               "${archive_name:?}"/"$topdir"/
+    done
+
+    check_symlinks "$archive_name" ||
+        die "Found some broken symbolic links after combining\n"
+
+    # Rename sof-WIP -> sof-vX.Y, tools-WIP -> tools-vX.Y, ...
+    ( cd "${archive_name}"
+      set -e
+      for d in *-WIP; do
+          radix=${d%-WIP}
+          ( set -x; mv "$d" "${radix}-$last_ver")
+      done
+    )
+
+    local manifest_file="$archive_name"/manifest.txt
+    # Share which version was used for each file in manifest.txt
+    for f in "${!FILES_ORIGIN[@]}"; do
+        printf '%s\t%s\n' "${FILES_ORIGIN[$f]}" "$f"
+    done | sort -V > "$manifest_file"
+
+    printf '\n#   --- symbolic links ---\n\n' >> "$manifest_file"
+
+    for f in "${!SYMLINKS_ORIGIN[@]}"; do
+        printf '%s\t%s -> %s\n' \
+               "${SYMLINKS_ORIGIN[$f]}" "$f" "${SYMLINKS_TARGET[$f]}"
+    done | sort -V >> "$manifest_file"
+
+    for f in "${!FILES_CHKSUM[@]}"; do
+        topdir="${FILES_ORIGIN[$f]}"
+        ver=$(parse_version_suffix "$topdir")
+        radix=${topdir%-"$ver"}
+        printf '%s\t%s\n' "${FILES_CHKSUM[$f]}" "$radix-$last_ver/$f"
+    done | LC_ALL=C sort -k2 > "$archive_name"/sha256sum.txt
+
+    ( cd "${archive_name}"/; set -x
+      sha256sum --quiet --check sha256sum.txt
+    )
+
+    # Final tarball
+    (set -ex
+     tar cfz "$archive_name".tar.gz "$archive_name"/
+     rm -r "${archive_name:?}"/
+    )
+}
+
+# vX.Y.Z/tools-v1.2.3   ->  v1.2.3
+parse_version_suffix()
+{
+    local arg="$1" base version
+
+    base=$(basename "$arg")
+    # Remove everything before the last '-v'
+    version=${base##*-v}
+
+    if [ "$version" = "$base" ]; then
+        die 'failed to find a version in "%s"\n' "$base"
+    fi
+
+    # Add the 'v' back
+    printf 'v%s' "$version"
+}
+
+parse_args()
+{
+    GIT_REF=HEAD
+    local opt
+    while getopts "g:h" opt; do
+        case "$opt" in
+            g) GIT_REF=$OPTARG ;;
+            h) usage ;;
+            *) exit 1;;
+        esac
+    done
+    shift $((OPTIND-1))
+
+    TOP_DIRS=("$@")
+    local dir
+    for dir in "${TOP_DIRS[@]}"; do
+        test -d "$gittop/$dir" ||
+            die "%s is not a directory in %s\n" "$dir" "$gittop"
+        # Better fail earlier
+        parse_version_suffix "$dir" > /dev/null
+    done
+}
+
+declare -A FILES_ORIGIN FILES_CHKSUM SYMLINKS_ORIGIN SYMLINKS_TARGET
+record_files_version()
+{
+    local f root="$1" topdir="$2" ver="$3"
+    pushd "$root"/"$topdir" > /dev/null
+
+    # shell check is right but the alternatives it suggests do not let
+    # us define variables / side-effects. We could probably write to a
+    # file and source it... awkward.
+    # shellcheck disable=SC2044
+    for f in $(find . -type f); do
+        FILES_ORIGIN["$f"]="$topdir"
+        FILES_CHKSUM["$f"]=$(sha256sum "$f" | awk '{ print $1 }')
+    done
+    # Keep symbolic links separate
+    # shellcheck disable=SC2044
+    for f in $(find . -type l); do
+        SYMLINKS_ORIGIN["$f"]="$topdir"
+        SYMLINKS_TARGET["$f"]="$(readlink "$f")"
+    done
+    popd > /dev/null
+}
+
+# Look for broken symlinks in directory argument. Requires GNU find.
+check_symlinks()
+{
+    local dir="$1"
+    if find "${dir}"/ -xtype l | grep -q . ; then
+        # Show all symlinks
+        find "${dir}"/ -xtype l -exec file {} \;
+        return 1
+    fi
+    return 0
+}
+
+die()
+{
+    >&2 printf '%s ERROR: ' "$0"
+    # We want die() to be usable exactly like printf
+    # shellcheck disable=SC2059
+    >&2 printf "$@"
+    exit 1
+}
+
+main "$@"

tests/common_helpers.bash

@@ -6,6 +6,8 @@ set_constants()
     TOP_DIR=$(cd "${TESTS_DIR}"/.. && pwd)
     REFS="$TESTS_DIR"/refs/
     EXTR_REFS="$TESTS_DIR"/extracted_refs/
+    # shellcheck disable=SC2034
+    STATIC_REFS="$TESTS_DIR"/static_refs/
 
     GITHUB_RELEASES='https://github.com/thesofproject/sof-bin/releases/download'
 }

tests/static_refs/multi-v2.1.1a/manifest.txt

@@ -0,0 +1,74 @@
+sof-tplg-v2.1.1a	./sof-adl-rt711-l0-rt1308-l12-rt715-l3.tplg
+sof-tplg-v2.1.1a	./sof-adl-rt711-l0-rt1316-l12-rt714-l3.tplg
+sof-tplg-v2.1.1a	./sof-adl-rt711-l0-rt1316-l13-rt714-l2.tplg
+sof-tplg-v2.1.1a	./sof-adl-rt711-l2-rt1316-l01.tplg
+sof-tplg-v2.1.1a	./sof-adl-rt711-l2-rt1316-l01-rt714-l3.tplg
+sof-tplg-v2.1.1a	./sof-adl-rt1316-l2-mono-rt714-l0.tplg
+sof-tplg-v2.1.1a	./sof-adl-rt1316-l2-mono-rt714-l3.tplg
+sof-tplg-v2.1.1a	./sof-adl-rt1316-l12-rt714-l0.tplg
+sof-tplg-v2.1.1a	./sof-cml-rt711-rt1308-mono-rt715.tplg
+sof-tplg-v2.1.1a	./sof-cml-rt711-rt1308-rt715.tplg
+sof-tplg-v2.1.1a	./sof-icl-rt711-rt1308-rt715.tplg
+sof-tplg-v2.1.1a	./sof-tgl-rt711-l0-rt1316-l1-mono-rt714-l3.tplg
+sof-tplg-v2.1.1a	./sof-tgl-rt711-rt1308-mono-rt715.tplg
+sof-tplg-v2.1.1a	./sof-tgl-rt711-rt1308-rt715.tplg
+sof-tplg-v2.1.1a	./sof-tgl-rt711-rt1316-rt714.tplg
+sof-tplg-v2.1.1a	./sof-tgl-rt715-rt711-rt1308-mono.tplg
+sof-v1.8-rc2	./community/sof-apl.ri
+sof-v1.8-rc2	./community/sof-cnl.ri
+sof-v1.8-rc2	./community/sof-icl.ri
+sof-v1.8-rc2	./community/sof-jsl.ri
+sof-v1.8-rc2	./community/sof-tgl.ri
+sof-v1.8-rc2	./community/sof-tgl-h.ri
+sof-v1.8-rc2	./intel-signed/sof-apl.ri
+sof-v1.8-rc2	./intel-signed/sof-cnl.ri
+sof-v1.8-rc2	./intel-signed/sof-ehl.ri
+sof-v1.8-rc2	./intel-signed/sof-icl.ri
+sof-v1.8-rc2	./intel-signed/sof-jsl.ri
+sof-v1.8-rc2	./intel-signed/sof-tgl.ri
+sof-v1.8-rc2	./intel-signed/sof-tgl-h.ri
+sof-v1.8-rc2	./sof-apl.ldc
+sof-v1.8-rc2	./sof-bdw.ldc
+sof-v1.8-rc2	./sof-bdw.ri
+sof-v1.8-rc2	./sof-byt.ldc
+sof-v1.8-rc2	./sof-byt.ri
+sof-v1.8-rc2	./sof-cht.ldc
+sof-v1.8-rc2	./sof-cht.ri
+sof-v1.8-rc2	./sof-cnl.ldc
+sof-v1.8-rc2	./sof-icl.ldc
+sof-v1.8-rc2	./sof-jsl.ldc
+sof-v1.8-rc2	./sof-tgl.ldc
+sof-v1.8-rc2	./sof-tgl-h.ldc
+sof-v2.2	./intel-signed/sof-adl.ri
+sof-v2.2	./intel-signed/sof-adl-s.ri
+tools-v1.9-rc1	./sof-logger
+
+#   --- symbolic links ---
+
+sof-v1.8-rc2	./community/sof-cfl.ri -> sof-cnl.ri
+sof-v1.8-rc2	./community/sof-cml.ri -> sof-cnl.ri
+sof-v1.8-rc2	./community/sof-ehl.ri -> sof-tgl.ri
+sof-v1.8-rc2	./community/sof-glk.ri -> sof-apl.ri
+sof-v1.8-rc2	./intel-signed/sof-cfl.ri -> sof-cnl.ri
+sof-v1.8-rc2	./intel-signed/sof-cml.ri -> sof-cnl.ri
+sof-v1.8-rc2	./intel-signed/sof-glk.ri -> sof-apl.ri
+sof-v1.8-rc2	./sof-apl.ri -> intel-signed/sof-apl.ri
+sof-v1.8-rc2	./sof-cfl.ldc -> sof-cnl.ldc
+sof-v1.8-rc2	./sof-cfl.ri -> intel-signed/sof-cfl.ri
+sof-v1.8-rc2	./sof-cml.ldc -> sof-cnl.ldc
+sof-v1.8-rc2	./sof-cml.ri -> intel-signed/sof-cml.ri
+sof-v1.8-rc2	./sof-cnl.ri -> intel-signed/sof-cnl.ri
+sof-v1.8-rc2	./sof-ehl.ldc -> sof-tgl.ldc
+sof-v1.8-rc2	./sof-ehl.ri -> intel-signed/sof-ehl.ri
+sof-v1.8-rc2	./sof-glk.ldc -> sof-apl.ldc
+sof-v1.8-rc2	./sof-glk.ri -> intel-signed/sof-glk.ri
+sof-v1.8-rc2	./sof-icl.ri -> intel-signed/sof-icl.ri
+sof-v1.8-rc2	./sof-jsl.ri -> intel-signed/sof-jsl.ri
+sof-v1.8-rc2	./sof-tgl-h.ri -> intel-signed/sof-tgl-h.ri
+sof-v1.8-rc2	./sof-tgl.ri -> intel-signed/sof-tgl.ri
+sof-v2.2	./community/sof-adl-s.ri -> sof-tgl-h.ri
+sof-v2.2	./community/sof-adl.ri -> sof-tgl.ri
+sof-v2.2	./sof-adl-s.ldc -> sof-tgl-h.ldc
+sof-v2.2	./sof-adl-s.ri -> intel-signed/sof-adl-s.ri
+sof-v2.2	./sof-adl.ldc -> sof-tgl.ldc
+sof-v2.2	./sof-adl.ri -> intel-signed/sof-adl.ri

tests/static_refs/multi-v2.1.1a/sha256sum.txt

@@ -0,0 +1,44 @@
+076b5b8a7a85016915951f2cb01b9591e8eaad53d143bf827be3ec2990cc9559	sof-tplg-v2.1.1a/./sof-adl-rt1316-l12-rt714-l0.tplg
+a37b0b005930ba7348f7281c7351e78e602c50a1f6b7012a9df8e168b2dd63a5	sof-tplg-v2.1.1a/./sof-adl-rt1316-l2-mono-rt714-l0.tplg
+4ca91811d7bc7b157739ce6ad79e9f4197e17c39583d60976fde45ae2dda03be	sof-tplg-v2.1.1a/./sof-adl-rt1316-l2-mono-rt714-l3.tplg
+1a043202b7d8e24735996ec0de8f9353303c57c13cdbef5770504b1ebc33af5a	sof-tplg-v2.1.1a/./sof-adl-rt711-l0-rt1308-l12-rt715-l3.tplg
+1a043202b7d8e24735996ec0de8f9353303c57c13cdbef5770504b1ebc33af5a	sof-tplg-v2.1.1a/./sof-adl-rt711-l0-rt1316-l12-rt714-l3.tplg
+2d7132baa504e9a588f626bb8b865d78b19c98cb1923a03c8d2cab752d228e3c	sof-tplg-v2.1.1a/./sof-adl-rt711-l0-rt1316-l13-rt714-l2.tplg
+11e07fa6d37e726127559cc6799ef7f73aff04868adf0d28a4d0aaa7a2c8cbc1	sof-tplg-v2.1.1a/./sof-adl-rt711-l2-rt1316-l01-rt714-l3.tplg
+85281ce60cd07c0b7d541d43782505f896064f75272e45566847e0f9f5cdb02f	sof-tplg-v2.1.1a/./sof-adl-rt711-l2-rt1316-l01.tplg
+3af494e1092c876155e05e04262c528f92257aa22e767c4dc409883ffca229ee	sof-tplg-v2.1.1a/./sof-cml-rt711-rt1308-mono-rt715.tplg
+1a043202b7d8e24735996ec0de8f9353303c57c13cdbef5770504b1ebc33af5a	sof-tplg-v2.1.1a/./sof-cml-rt711-rt1308-rt715.tplg
+1a043202b7d8e24735996ec0de8f9353303c57c13cdbef5770504b1ebc33af5a	sof-tplg-v2.1.1a/./sof-icl-rt711-rt1308-rt715.tplg
+3af494e1092c876155e05e04262c528f92257aa22e767c4dc409883ffca229ee	sof-tplg-v2.1.1a/./sof-tgl-rt711-l0-rt1316-l1-mono-rt714-l3.tplg
+3af494e1092c876155e05e04262c528f92257aa22e767c4dc409883ffca229ee	sof-tplg-v2.1.1a/./sof-tgl-rt711-rt1308-mono-rt715.tplg
+1a043202b7d8e24735996ec0de8f9353303c57c13cdbef5770504b1ebc33af5a	sof-tplg-v2.1.1a/./sof-tgl-rt711-rt1308-rt715.tplg
+1a043202b7d8e24735996ec0de8f9353303c57c13cdbef5770504b1ebc33af5a	sof-tplg-v2.1.1a/./sof-tgl-rt711-rt1316-rt714.tplg
+b413c1fbda747034c3f796e7ff4da10ba985c7fe2062b5087233666a3212dbad	sof-tplg-v2.1.1a/./sof-tgl-rt715-rt711-rt1308-mono.tplg
+ed32cc8e67d453da7873a0e0ee43a26e0447b83a0b980a5f62a5ab600cac2f34	sof-v2.1.1a/./community/sof-apl.ri
+310799f69388a58881f9c0ded25496e4789fad714157ed7261d197e768069914	sof-v2.1.1a/./community/sof-cnl.ri
+1d0a5fc5aba09359744c6e596c7eb919728f637665130a8082993f1322cb68df	sof-v2.1.1a/./community/sof-icl.ri
+e20540a41ac8be7718ea17263f495824ab6ca04ec5062c0048942ef69d10d29b	sof-v2.1.1a/./community/sof-jsl.ri
+2ab7fc689cc7f560bd209fbdcd6e5c38e536e9b9435963943a34cdd34df990d4	sof-v2.1.1a/./community/sof-tgl-h.ri
+700848108549ea32c020fa506d58a75f8c03adf6fd014600edb9de4858d7e8ef	sof-v2.1.1a/./community/sof-tgl.ri
+1bddc1c6e26de3ae37f914305a0d619fa51e1b2fa3ebc5a64550b3464cf13188	sof-v2.1.1a/./intel-signed/sof-adl-s.ri
+294e5a51b1d3fc4ceda57780927c2a2ec95bb5d23477f879a9fb648c6c01cb30	sof-v2.1.1a/./intel-signed/sof-adl.ri
+4d97921846fe979b1295cfd46a62f159b118e70e9aa8fefe91820a7a2431d018	sof-v2.1.1a/./intel-signed/sof-apl.ri
+3e23a6fa70dbeea2a932ac38ae2d93e3041566f68f1b10d7b9b15806c9348306	sof-v2.1.1a/./intel-signed/sof-cnl.ri
+acb16f18f2e4f572ad51865bc8e783f085d0e4188f2b012db3f28550099261ce	sof-v2.1.1a/./intel-signed/sof-ehl.ri
+59aa8acab8ec4e55c69c6a0739330b538e49b5ba14c1743a814dd04f7ba571d8	sof-v2.1.1a/./intel-signed/sof-icl.ri
+0b19633207ac7d4c606586b53eb5b0934e08fe40e4eab9b37a777a24e83e86f1	sof-v2.1.1a/./intel-signed/sof-jsl.ri
+f74c1c3586a73df0dd469b16d5e283178579c6f1a8f59667524a7b20bfa8ec52	sof-v2.1.1a/./intel-signed/sof-tgl-h.ri
+d94a423b3eceaf4d28821159f08e95b4e66b8c457c20fa80c2b57015d561905c	sof-v2.1.1a/./intel-signed/sof-tgl.ri
+e8b5e4fa027ea880baec228b7454cfe937ff91938038cc6ea6d111da7f1d5049	sof-v2.1.1a/./sof-apl.ldc
+2081977784980805f517c265befc3f75532daacbfcf06cabd29084ea0e968b08	sof-v2.1.1a/./sof-bdw.ldc
+d78f313b33a63680b4608431029788b52c90d5af8c368a14539e2f025cce5062	sof-v2.1.1a/./sof-bdw.ri
+ea34fdc893c1e8a7ab33ac8ee387c88dd55894ba1b5b167590b870b62a95c382	sof-v2.1.1a/./sof-byt.ldc
+7ae21ed51bbfd91792f30ae2121c0886e22a3ea00aba2ee92e17d585b38613d2	sof-v2.1.1a/./sof-byt.ri
+ea34fdc893c1e8a7ab33ac8ee387c88dd55894ba1b5b167590b870b62a95c382	sof-v2.1.1a/./sof-cht.ldc
+870b833f3d02e04d0e5ed30f4a78c73566899fc86e1f65ed0a6fd4f468baead2	sof-v2.1.1a/./sof-cht.ri
+0eec9d9aba71bee01caadd2b4801e085c0527beef6ca39da47346d57b94febe2	sof-v2.1.1a/./sof-cnl.ldc
+ba82bdbc813714939cebf9428cd9395488d39958b15a5afe56b48a699390f024	sof-v2.1.1a/./sof-icl.ldc
+a7edc72857cf0a8f80b80034c9db6d5928630e0884bdffbe51bb060078c59157	sof-v2.1.1a/./sof-jsl.ldc
+6bd47e2b8f0eeb0427b38ca7674277a91c0d864cfcf7063c17837c8d0e05f2ea	sof-v2.1.1a/./sof-tgl-h.ldc
+6bd47e2b8f0eeb0427b38ca7674277a91c0d864cfcf7063c17837c8d0e05f2ea	sof-v2.1.1a/./sof-tgl.ldc
+96fe201e988d935c458ea1fd6f5712555ddeee1d96bcdfb1cd9f0045c3f8925f	tools-v2.1.1a/./sof-logger

tests/tests.bats

@@ -49,6 +49,20 @@ teardown()
     test_tarball_topologies_only v1.9.x v1.9.3-tplg2
 }
 
+@test "tarball_multi_2_1_1a" {
+    pushd "$RUN_DIR"/ || exit 1
+    load 'common_helpers.bash';  set_constants
+
+    "$TOP_DIR"/tarball_multi_releases.bash \
+      v2.2.x/sof-v2.2  v1.8.x/sof-v1.8-rc2  v1.9.x/tools-v1.9-rc1/ \
+      v2.1.x/sof-tplg-v2.1.1a
+    tar xf sof-bin-v2.1.1a.tar.gz
+    for i in manifest.txt sha256sum.txt; do
+        diff -u "$STATIC_REFS"/multi-v2.1.1a/"$i" "$(pwd)/sof-bin-v2.1.1a/$i"
+    done
+    popd || exit 1
+}
+
 test_tarball_one_version()
 {
     local dir="$1" ver="$2"