Skip to content

[CIEXE-2681] Use published Docker CI images with internal Buildx #882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ericdog wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[CIEXE-2681] Use published Docker CI images with internal Buildx #882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions ci/input_files/build.yaml.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ default:
# Retry when the runner fails to start
- runner_system_failure

# The Noble Python image uses managed system Python and does not include AWS CLI.
.install-python-tools: &install-python-tools
- apt-get update
- apt-get install -y --no-install-recommends awscli python3-venv python-is-python3

# This is for serverless framework
.install-node: &install-node
- apt-get update
Expand All @@ -38,7 +43,7 @@ default:
build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
artifacts:
expire_in: 1 day # Unsigned zips expire in 1 day
paths:
Expand All @@ -53,7 +58,7 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
needs:
- build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
dependencies:
Expand Down Expand Up @@ -99,7 +104,7 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
rules:
- if: '$SKIP_E2E_TESTS == "true"'
when: never
Expand All @@ -112,6 +117,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
variables:
CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
before_script:
- *install-python-tools
- *install-node
- EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
- yarn global add serverless@^3.38.0 --prefix /usr/local
Expand All @@ -124,7 +130,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: sign
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
when: manual
Expand All @@ -141,6 +147,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
paths:
- .layers/datadog_lambda_py-{{ $runtime.arch }}-{{ $runtime.python_version }}.zip
before_script:
- *install-python-tools
- apt-get update
- apt-get install -y uuid-runtime
{{ with $environment := (ds "environments").environments.prod }}
Expand All @@ -155,7 +162,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
rules:
- if: '$SKIP_E2E_TESTS == "true"'
when: never
Expand Down Expand Up @@ -190,6 +197,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
- {{ .code }}
{{- end}}
before_script:
- *install-python-tools

@datadog-datadog-prod-us1-2 datadog-datadog-prod-us1-2 Bot Nov 4, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Skip the redundant system-pip install in layer-publishing jobs

With SKIP_PIP_INSTALL unset, publish-layer-* jobs still execute pip install awscli in ci/publish_layers.sh despite installing AWS CLI through apt here. Noble's externally managed Python rejects that installation, and set -e aborts publishing before any layers are published. Export the existing skip flag for these jobs.

Suggested change
- *install-python-tools
- *install-python-tools
- export SKIP_PIP_INSTALL=true

React: 👍 useful · 👎 wrong · 😕 unclear · 👀 right, but not acting on it

What each reaction means
  • 👍 Useful: correct, and you're fixing it or will.
  • 👎 Wrong: the finding is incorrect, for example the bug can't happen or the code was misread.
  • 😕 Unclear: you can't tell what it means or why it matters.
  • 👀 Right, not acting: correct, but intended, out of scope for this PR, a known risk, or too minor.

🤖 Bits Code Review · Open Bits AI session

- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
script:
- STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh
Expand All @@ -201,8 +209,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-pypi-package:
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
before_script:
- *install-python-tools
- ./scripts/setup_python_env.sh
cache: []
rules:
Expand All @@ -217,7 +226,7 @@ publish-pypi-package:
layer bundle:
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
needs:
{{ range (ds "runtimes").runtimes }}
- build-layer ({{ .name }}-{{ .arch }})
Expand All @@ -238,7 +247,7 @@ layer bundle:

signed layer bundle:
stage: sign
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
tags: ["arch:amd64"]
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
Expand Down Expand Up @@ -286,7 +295,7 @@ e2e-test:

e2e-test-status:
stage: e2e
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
tags: ["arch:amd64"]
timeout: 3h
rules:
Expand Down
3 changes: 1 addition & 2 deletions scripts/setup_python_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ if [ -z "${IS_LINT:-}" ] || [ "${IS_LINT}" = "0" ] || [ "${IS_LINT}" = "false" ]
spec_ddtrace_dep
fi

pip install virtualenv
virtualenv venv
python3 -m venv venv
source venv/bin/activate
pip install .[dev]
pip install poetry
Expand Down
Loading