From 5a43c1139333d05a9e5ddbba4e876b4f180e0884 Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Mon, 3 Jan 2022 21:17:08 +0200 Subject: [PATCH 1/6] xtensa-build-zephyr: add -v option for verbose build Align with xtensa-build-all.sh and add a "-v" option to toggle verbose build log ("west -v build"). Signed-off-by: Kai Vehmanen (cherry picked from commit b95d995bd2fb1f4f2b13575f11e5a42e18ad110e) Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/xtensa-build-zephyr.sh b/scripts/xtensa-build-zephyr.sh index 52165b7e9230..7e98291850cb 100755 --- a/scripts/xtensa-build-zephyr.sh +++ b/scripts/xtensa-build-zephyr.sh @@ -74,6 +74,8 @@ usage: $0 [options] [ platform(s) ] [ -- cmake arguments ] sof/ and zephyr-project/ directories are not nested in one another. + -v Verbose Makefile log + This script supports XtensaTools but only when installed in a specific directory structure, example: @@ -244,6 +246,9 @@ build_platforms() # show a confusing error. /bin/pwd + local verbose + [ -z "$BUILD_VERBOSE" ] || verbose=-v + if test -e "$bdir/build.ninja" || test -e "$bdir/Makefile"; then test -z "${CMAKE_ARGS+defined}" || die 'Cannot re-define CMAKE_ARGS, you must delete %s first\n' \ @@ -252,10 +257,10 @@ build_platforms() # CMAKE_ARGS this _not_ does force CMake to re-run and passing # a different board by mistake is very nicely caught by west. set -x - west build --build-dir "$bdir" --board "$PLAT_CONFIG" + west $verbose build --build-dir "$bdir" --board "$PLAT_CONFIG" else set -x - west build --build-dir "$bdir" --board "$PLAT_CONFIG" \ + west $verbose build --build-dir "$bdir" --board "$PLAT_CONFIG" \ zephyr/samples/subsys/audio/sof \ -- "${CMAKE_ARGS[@]}" fi @@ -301,7 +306,7 @@ parse_args() local OPTIND=1 # Parse -options - while getopts "acz:j:k:p:" OPTION; do + while getopts "acz:j:k:p:v" OPTION; do case "$OPTION" in a) PLATFORMS=("${DEFAULT_PLATFORMS[@]}") ;; c) DO_CLONE=yes ;; @@ -309,6 +314,7 @@ parse_args() j) BUILD_JOBS="$OPTARG" ;; k) RIMAGE_KEY_OPT="$OPTARG" ;; p) zeproj="$OPTARG" ;; + v) BUILD_VERBOSE=true ;; *) print_usage; exit 1 ;; esac done From b4c7f5e264882c97165acdd2950d08907879effd Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 11 Mar 2022 22:49:17 -0800 Subject: [PATCH 2/6] xtensa-build-zephyr.sh: add new -u zephyr_remote option Same -u option as the new Python script. We need this for the new sof/stable-v2.1 branch on https://github.com/thesofproject/zephyr while our CI has not entirely switched to the python script yet. Signed-off-by: Marc Herbert (cherry picked from commit 177e18f701ef625fd780bfeb9d88b59fb44b3157) --- scripts/xtensa-build-zephyr.sh | 38 ++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/scripts/xtensa-build-zephyr.sh b/scripts/xtensa-build-zephyr.sh index 7e98291850cb..9ebb29cc2a57 100755 --- a/scripts/xtensa-build-zephyr.sh +++ b/scripts/xtensa-build-zephyr.sh @@ -59,6 +59,9 @@ usage: $0 [options] [ platform(s) ] [ -- cmake arguments ] Incompatible with -p. To stop after downloading Zephyr, do not pass any platform or cmake argument. + -u Initial Zephyr remote for the -c option. Default value: + https://github.com/zephyrproject-rtos/zephyr + -z Initial Zephyr git ref for the -c option. Can be a branch, tag, full SHA1 in a fork, magic pull/12345/merge,... anything as long as it is fetchable from https://github.com/zephyrproject-rtos/zephyr/ @@ -113,18 +116,26 @@ zephyr_fetch_and_switch() # link back to sof/ west_init_update() { - local init_ref="$1" - - # Or, we could have a default value: - # init_ref=${1:-811a09bd8305} - - git clone --depth=5 https://github.com/zephyrproject-rtos/zephyr \ - "$WEST_TOP"/zephyr + # Default Zephyr remote and branch if nothing passed on the + # command line. + local init_remote="${1:-https://github.com/zephyrproject-rtos/zephyr}" + + # This can be a branch, a 40-digit SHA or empty to fetch the + # default branch. Example: + # local init_ref=${2:-sof/stable-v2.1} + local init_ref="$2" + + # git fetch accepts anything, even 40-digits SHA but git clone is + # less flexible. So we git clone the default branch first to get + # started and then we switch to any 'z_ref' thanks to git fetch. + ( set -x + git clone --depth=5 "$init_remote" "$WEST_TOP"/zephyr + ) # To keep things simple, this moves to a detached HEAD even when # init_ref is a (remote) branch. test -z "$init_ref" || - zephyr_fetch_and_switch origin "${init_ref}" + zephyr_fetch_and_switch "${init_remote}" "${init_ref}" # This shows how to point CI at any Zephyr commit from anywhere # and to run all tests on it. Simply edit remote and reference, @@ -302,14 +313,15 @@ build_platforms() parse_args() { local zeproj - unset zephyr_ref + unset zephyr_remote zephyr_ref local OPTIND=1 # Parse -options - while getopts "acz:j:k:p:v" OPTION; do + while getopts "acu:z:j:k:p:v" OPTION; do case "$OPTION" in a) PLATFORMS=("${DEFAULT_PLATFORMS[@]}") ;; c) DO_CLONE=yes ;; + u) zephyr_remote="$OPTARG" ;; z) zephyr_ref="$OPTARG" ;; j) BUILD_JOBS="$OPTARG" ;; k) RIMAGE_KEY_OPT="$OPTARG" ;; @@ -328,8 +340,8 @@ parse_args() die 'Cannot use -p with -c, -c supports %s only' "${SOF_TOP}/zephyrproject" fi - if [ -n "$zephyr_ref" ] && [ -z "$DO_CLONE" ]; then - die '%s' '-z without -c makes no sense' + if [ -n "$zephyr_remote" -o -n "$zephyr_ref" ] && [ -z "$DO_CLONE" ]; then + die '%s' '-u or -z without -c makes no sense' fi if [ -n "$zeproj" ]; then @@ -407,7 +419,7 @@ see https://docs.zephyrproject.org/latest/getting_started/index.html" # Resolve symlinks mkdir "$zep"; WEST_TOP=$( cd "$zep" && /bin/pwd ) - west_init_update "${zephyr_ref}" + west_init_update "${zephyr_remote}" "${zephyr_ref}" else # Look for Zephyr and define WEST_TOP From 5669379d790468876425d3da79798df56021ae92 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 8 Apr 2022 14:33:32 -0700 Subject: [PATCH 3/6] xtensa-build: set default value instead of hardcoding Zephyr commit This reverts commit b678a4dcaa1e7ad0c6a056e17dd38bfe095a4464 because it's hardcoding the zephyr version real hard. Instead, merely change the default value. Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/xtensa-build-zephyr.sh b/scripts/xtensa-build-zephyr.sh index 9ebb29cc2a57..d109f274b470 100755 --- a/scripts/xtensa-build-zephyr.sh +++ b/scripts/xtensa-build-zephyr.sh @@ -121,9 +121,8 @@ west_init_update() local init_remote="${1:-https://github.com/zephyrproject-rtos/zephyr}" # This can be a branch, a 40-digit SHA or empty to fetch the - # default branch. Example: - # local init_ref=${2:-sof/stable-v2.1} - local init_ref="$2" + # default branch + local init_ref="${2:-fd089b361d8aebbcb49cca989444aaeb0b6fbb4d}" # git fetch accepts anything, even 40-digits SHA but git clone is # less flexible. So we git clone the default branch first to get @@ -147,9 +146,6 @@ west_init_update() # zephyr_fetch_and_switch origin pull/38374/head # zephyr_fetch_and_switch origin 19d5448ec117fde8076bec4d0e61da53147f3315 - # SOF2.0 release uses this commit, verified in 2021-12-03 daily test - zephyr_fetch_and_switch origin fd089b361d8aebbcb49cca989444aaeb0b6fbb4d - # SECURITY WARNING for reviewers: never allow unknown code from # unknown submitters on any CI system. From 1fbd2e3f541e8b683647ada49929efb2a08fae29 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 27 Apr 2022 00:58:31 +0000 Subject: [PATCH 4/6] xtensa-build-zephyr.sh: switch to new Zephyr branch sof/stable-v2.0 The brand new sof/stable-v2.0 branch has (for now) only these two commits that restore compatibility with the 0.14.0 Zephyr SDK and the latest zephyr-build container 51d2a25590cc "Revert cmake: Zephyr sdk backward compatibility with 0.11.1 and 0.11.2" af345ec33aa2 (sof/sof/stable-v2.0) cmake: remove xtensa workaround in Zephyr toolchain code. See longer story in review #5654 for commit 7ec25f16af9e Signed-off-by: Marc Herbert (cherry picked from commit f8486cca4ec3a98e9df7aaf2e92c8c95036c9f15) --- scripts/xtensa-build-zephyr.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/xtensa-build-zephyr.sh b/scripts/xtensa-build-zephyr.sh index d109f274b470..a6f3b5a74fd0 100755 --- a/scripts/xtensa-build-zephyr.sh +++ b/scripts/xtensa-build-zephyr.sh @@ -118,11 +118,11 @@ west_init_update() { # Default Zephyr remote and branch if nothing passed on the # command line. - local init_remote="${1:-https://github.com/zephyrproject-rtos/zephyr}" + local init_remote="${1:-https://github.com/thesofproject/zephyr}" # This can be a branch, a 40-digit SHA or empty to fetch the # default branch - local init_ref="${2:-fd089b361d8aebbcb49cca989444aaeb0b6fbb4d}" + local init_ref="${2:-sof/stable-v2.0}" # git fetch accepts anything, even 40-digits SHA but git clone is # less flexible. So we git clone the default branch first to get From 0bdfafbc7241321e29e94a6f0e905eee151b2e65 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Thu, 6 Jan 2022 22:49:09 +0000 Subject: [PATCH 5/6] docker-run.sh: log command run and image digests There's frequent confusion between image ID and image digest: display both. Show both sof and thesofproject/sof. Use set -x to display the full command that is run. Signed-off-by: Marc Herbert (cherry picked from commit f79d2341af3c6c3f025adadebc7c88bb5a4c8c28) --- scripts/docker-run.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh index 597122a4cd99..40886314af09 100755 --- a/scripts/docker-run.sh +++ b/scripts/docker-run.sh @@ -16,14 +16,16 @@ set -e SOF_TOP="$(cd "$(dirname "$0")"/.. && /bin/pwd)" -# Log the container version. Especially useful when forgetting to update -# the 'sof' shortcut. -docker images sof +# Log container versions +for rep in sof thesofproject/sof; do + docker images --digests "$rep" +done if tty --quiet; then SOF_DOCKER_RUN="$SOF_DOCKER_RUN --tty" fi +set -x docker run -i -v "${SOF_TOP}":/home/sof/work/sof.git \ -v "${SOF_TOP}":/home/sof/work/sof-bind-mount-DO-NOT-DELETE \ --env CMAKE_BUILD_TYPE \ From 6ccef7b3d42437a209c821166dad9e21ea696e91 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Mon, 25 Apr 2022 16:30:18 -0700 Subject: [PATCH 6/6] zephyr/docker-build.sh: match UID with 'adduser' instead of 'chgrp -R' Some conflicts because this branch does not have the new xtensa-build-zephyr.py, it still uses xtensa-build-zephyr.sh This fixes SOF version.cmake which was just broken by a recent git security update and started to fail like this: ``` -- SOF version.cmake starting at 2022-04-25T18:14:56Z UTC -- /workdir/zephyr/.. is at git commit with parent(s): fatal: unsafe repository ('/workdir' is owned by someone else) To add an exception for this directory, call: git config --global --add safe.directory /workdir ``` (example at https://github.com/thesofproject/sof/runs/6162885265) chgrp -R was always an ugly hack because it was messing with (persistent) file permissions on the host, outside the container. This new adduser solution is unfortunately much more code but it does not leak any side effect outside the container. Do not fix scripts/docker-run.sh yet because there is still no UID mismatch between Github Actions and the SOF container (they're both 1001) but add a warning + TODO. Signed-off-by: Marc Herbert (cherry picked from commit d09844ab98cc0f03681d2edbc0395a4f3e1a17ee) --- scripts/docker-run.sh | 5 ++++ zephyr/docker-build.sh | 53 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh index 40886314af09..ef674a53a855 100755 --- a/scripts/docker-run.sh +++ b/scripts/docker-run.sh @@ -25,6 +25,11 @@ if tty --quiet; then SOF_DOCKER_RUN="$SOF_DOCKER_RUN --tty" fi +# Not fatal, just a warning to allow other "creative" solutions. +# TODO: fix this with 'adduser' like in zephyr/docker-build.sh +test "$(id -n)" = 1001 || + >&2 printf "Warning: this script should be run as user ID 1001 to match the container\n" + set -x docker run -i -v "${SOF_TOP}":/home/sof/work/sof.git \ -v "${SOF_TOP}":/home/sof/work/sof-bind-mount-DO-NOT-DELETE \ diff --git a/zephyr/docker-build.sh b/zephyr/docker-build.sh index 0700cbaa780b..2a36f94befb4 100755 --- a/zephyr/docker-build.sh +++ b/zephyr/docker-build.sh @@ -12,12 +12,58 @@ set -x unset ZEPHYR_BASE -# Make sure we're in the right place; chgrp -R below. +# Make sure we're in the right place. test -e ./scripts/xtensa-build-zephyr.sh sudo apt-get update sudo apt-get -y install tree + +# See https://stackoverflow.com/questions/35291520/docker-and-userns-remap-how-to-manage-volume-permissions-to-share-data-betwee + many others +exec_as_sof_uid() +{ + local sof_uid; sof_uid="$(stat --printf='%u' .)" + local current_uid; current_uid="$(id -u)" + if test "$current_uid" = "$sof_uid"; then + return 0 + fi + + # Add new container user matching the host user owning the SOF + # checkout + local sof_user; sof_user="$(id "$sof_uid")" || { + sof_user=sof_zephyr_docker_builder + + local sof_guid; sof_guid="$(stat --printf='%g' .)" + + getent group "$sof_guid" || + sudo groupadd -g "$sof_guid" sof_zephyr_docker_group + + sudo useradd -m -u "$sof_uid" -g "$sof_guid" "$sof_user" + + local current_user; current_user="$(id -un)" + + # Copy sudo permissions just in case the build needs it + sudo sed -e "s/$current_user/$sof_user/" /etc/sudoers.d/"$current_user" | + sudo tee -a /etc/sudoers.d/"$sof_user" + sudo chmod --reference=/etc/sudoers.d/"$current_user" \ + /etc/sudoers.d/"$sof_user" + } + + # Safety delay: slower infinite loops are much better + sleep 0.5 + + # Double sudo to work around some funny restriction in + # zephyr-build:/etc/sudoers: 'user' can do anything but... only as + # root. + sudo sudo -u "$sof_user" "$0" "$@" + exit "$?" +} + +exec_as_sof_uid "$@" + +# Work in progress: move more code to a function +# https://github.com/thesofproject/sof-test/issues/740 + # As of container version 0.18.4, # https://github.com/zephyrproject-rtos/docker-image/blob/master/Dockerfile # installs two SDKs: ZSDK_VERSION=0.12.4 and ZSDK_ALT_VERSION=0.13.1 @@ -31,10 +77,5 @@ ln -s /opt/toolchains/zephyr-sdk-* ~/ if test -e zephyrproject; then ./scripts/xtensa-build-zephyr.sh -a "$@" else - # Matches docker.io/zephyrprojectrtos/zephyr-build:latest gid - ls -ln | head - stat . - sudo chgrp -R 1000 . - sudo chmod -R g+rwX . ./scripts/xtensa-build-zephyr.sh -a -c "$@" fi