Skip to content
Merged
97 changes: 28 additions & 69 deletions .github/workflows/build-ci-container-tooling.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,30 @@ on:
- '.github/workflows/containers/github-action-ci-tooling/**'
- llvm/utils/git/requirements_formatting.txt
- llvm/utils/git/requirements_linting.txt
- '.github/workflows/build-container/**'
- '.github/workflows/push-container/**'
pull_request:
paths:
- .github/workflows/build-ci-container-tooling.yml
- '.github/workflows/containers/github-action-ci-tooling/**'
- llvm/utils/git/requirements_formatting.txt
- llvm/utils/git/requirements_linting.txt
- '.github/workflows/build-container/**'
- '.github/workflows/push-container/**'

jobs:
build-ci-container-tooling:
name: Build Container ${{ matrix.container-name }}
if: github.repository_owner == 'llvm'
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
include:
- container-name: code-format
test-command: 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black'
- container-name: code-lint
test-command: 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage'
Comment on lines +34 to +38
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if it is possible, but could we give custom names for each include in matrix?

The resulting container names in UI are a bit odd because they include test-command:
build-ci-container-tooling (code-lint, cd $HOME && clang-tidy --version | grep version && clang-tidy...)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be possible by setting the name key explicitly for the job.

steps:
- name: Checkout LLVM
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
Expand All @@ -32,48 +45,15 @@ jobs:
llvm/utils/git/requirements_formatting.txt
llvm/utils/git/requirements_linting.txt
clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
.github/workflows/build-container

- name: Write Variables
id: vars
run: |
tag=$(git rev-parse --short=12 HEAD)
container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04"
echo "container-name-format=$container_name-code-format" >> $GITHUB_OUTPUT
echo "container-name-lint=$container_name-code-lint" >> $GITHUB_OUTPUT
echo "container-name-format-tag=$container_name-format:$tag" >> $GITHUB_OUTPUT
echo "container-name-lint-tag=$container_name-lint:$tag" >> $GITHUB_OUTPUT
echo "container-format-filename=$(echo $container_name-format:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
echo "container-lint-filename=$(echo $container_name-lint:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT

- name: Build container
run: |
podman build --target ci-container-code-format \
-f .github/workflows/containers/github-action-ci-tooling/Dockerfile \
-t ${{ steps.vars.outputs.container-name-format-tag }} .
podman build --target ci-container-code-lint \
-f .github/workflows/containers/github-action-ci-tooling/Dockerfile \
-t ${{ steps.vars.outputs.container-name-lint-tag }} .

# Save the container so we have it in case the push fails. This also
# allows us to separate the push step into a different job so we can
# maintain minimal permissions while building the container.
- name: Save container image
run: |
podman save ${{ steps.vars.outputs.container-name-format-tag }} > ${{ steps.vars.outputs.container-format-filename }}
podman save ${{ steps.vars.outputs.container-name-lint-tag }} > ${{ steps.vars.outputs.container-lint-filename }}

- name: Upload container image
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
- name: Build Container
uses: ./.github/workflows/build-container
with:
name: container-amd64
path: "*.tar"
retention-days: 14

- name: Test Container
run: |
# Use --pull=never to ensure we are testing the just built image.
podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-format-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black'
podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-lint-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage'
container-name: ci-ubuntu-24.04-${{ matrix.container-name }}
dockerfile: .github/workflows/containers/github-action-ci-tooling/Dockerfile
target: ci-container-${{ matrix.container-name }}
test-command: ${{ matrix.test-command }}

push-ci-container:
if: github.event_name == 'push'
Expand All @@ -82,34 +62,13 @@ jobs:
permissions:
packages: write
runs-on: ubuntu-24.04
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Download container
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0

- name: Push Container
run: |
function push_container {
image_name=$1
latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
podman tag $image_name $latest_name
echo "Pushing $image_name ..."
podman push $image_name
echo "Pushing $latest_name ..."
podman push $latest_name
}

podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
for f in $(find . -iname '*.tar'); do
image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
push_container $image_name
- name: Checkout LLVM
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
sparse-checkout: |
.github/workflows/push-container

if echo $image_name | grep '/amd64/'; then
# For amd64, create an alias with the arch component removed.
# This matches the convention used on dockerhub.
default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
podman tag $image_name $default_image_name
push_container $default_image_name
fi
done
- uses: ./.github/workflows/push-container
with:
token: ${{ secrets.GITHUB_TOKEN }}
107 changes: 31 additions & 76 deletions .github/workflows/build-ci-container.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,72 +10,46 @@ on:
paths:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'
- '.github/workflows/build-container/**'
- '.github/workflows/push-container/**'
pull_request:
paths:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'
- '.github/workflows/build-container/**'
- '.github/workflows/push-container/**'

jobs:
build-ci-container:
name: Build Container ${{ matrix.container-name }} ${{ (contains(matrix.runs-on, 'arm') && 'ARM64') || 'X64' }}
if: github.repository_owner == 'llvm'
runs-on: ${{ matrix.runs-on }}
strategy:
matrix:
include:
# The arch names should match the names used on dockerhub.
# See https://github.com/docker-library/official-images#architectures-other-than-amd64
- arch: amd64
runs-on: depot-ubuntu-24.04-16
- arch: arm64v8
runs-on: depot-ubuntu-24.04-arm-16
runs-on:
- depot-ubuntu-24.04-16
- depot-ubuntu-24.04-arm-16
container-name:
- ''
- agent
test-command:
- cd $HOME && printf '#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }' | clang++ -x c++ - && ./a.out | grep Hello
steps:
- name: Checkout LLVM
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
sparse-checkout: .github/workflows/containers/github-action-ci/
# podman is not installed by default on the ARM64 images.
- name: Install Podman
if: runner.arch == 'ARM64'
run: |
sudo apt-get install podman
- name: Write Variables
id: vars
run: |
tag=$(git rev-parse --short=12 HEAD)
container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/${{ matrix.arch }}/ci-ubuntu-24.04"
echo "container-name=$container_name" >> $GITHUB_OUTPUT
echo "container-name-agent=$container_name-agent" >> $GITHUB_OUTPUT
echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
echo "container-name-agent-tag=$container_name-agent:$tag" >> $GITHUB_OUTPUT
echo "container-filename=$(echo $container_name:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
echo "container-agent-filename=$(echo $container_name-agent:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT
- name: Build container
working-directory: ./.github/workflows/containers/github-action-ci/
run: |
podman build --target ci-container -t ${{ steps.vars.outputs.container-name-tag }} .
podman build --target ci-container-agent -t ${{ steps.vars.outputs.container-name-agent-tag }} .
sparse-checkout: |
.github/workflows/containers/github-action-ci/
.github/workflows/build-container

# Save the container so we have it in case the push fails. This also
# allows us to separate the push step into a different job so we can
# maintain minimal permissions while building the container.
- name: Save container image
run: |
podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }}
podman save ${{ steps.vars.outputs.container-name-agent-tag }} > ${{ steps.vars.outputs.container-agent-filename }}

- name: Upload container image
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
- name: Build Container
uses: ./.github/workflows/build-container
with:
name: container-${{ matrix.arch }}
path: "*.tar"
retention-days: 14

- name: Test Container
run: |
for image in ${{ steps.vars.outputs.container-name-tag }}; do
# Use --pull=never to ensure we are testing the just built image.
podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && printf '\''#include <iostream>\nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello'
done
container-name: ci-ubuntu-24.04${{ matrix.container-name && format('-{0}', matrix.container-name)}}
context: .github/workflows/containers/github-action-ci/
dockerfile: .github/workflows/containers/github-action-ci/Dockerfile
target: ci-container${{ matrix.container-name && format('-{0}', matrix.container-name) }}
test-command: ${{ matrix.test-command }}

push-ci-container:
if: github.event_name == 'push'
Expand All @@ -87,31 +61,12 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Download container
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0

- name: Push Container
run: |
function push_container {
image_name=$1
latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
podman tag $image_name $latest_name
echo "Pushing $image_name ..."
podman push $image_name
echo "Pushing $latest_name ..."
podman push $latest_name
}

podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
for f in $(find . -iname '*.tar'); do
image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
push_container $image_name
- name: Checkout LLVM
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
sparse-checkout: |
.github/workflows/push-container

if echo $image_name | grep '/amd64/'; then
# For amd64, create an alias with the arch component removed.
# This matches the convention used on dockerhub.
default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
podman tag $image_name $default_image_name
push_container $default_image_name
fi
done
- uses: ./.github/workflows/push-container
with:
token: ${{ secrets.GITHUB_TOKEN }}
95 changes: 95 additions & 0 deletions .github/workflows/build-container/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
name: Build Container
description: >-
Build and test a container using the standard llvm naming scheme for containers.
inputs:
tag:
description: >-
The tag to use for this container.
required: false
container-name:
description: >-
The name for the container.
required: true
dockerfile:
description: >-
Path to docker file.
required: false
target:
description: >-
The container target to build 'passed to podman via ---target option'
required: false
context:
description: >-
Path to context for the container build.
required: false
test-command:
description: >-
Test command to run to ensure the container is working correctly.
required: false

runs:
using: "composite"
steps:
# podman is not installed by default on the ARM64 images.
- name: Install Podman
if: runner.arch == 'ARM64'
shell: bash
run: |
sudo apt-get install podman
- name: Build Container
shell: bash
env:
INPUT_TAG: ${{inputs.tag }}
INPUT_CONTAINER_NAME: ${{ inputs.container-name }}
INPUT_TARGET: ${{ inputs.target }}
INPUT_DOCKERFILE: ${{ inputs.dockerfile }}
INPUT_CONTEXT: ${{ inputs.context }}
id: build
run: |
env
tag="${INPUT_TAG:-$(git rev-parse --short=12 HEAD)}"
case "$RUNNER_ARCH" in
ARM64)
container_arch="arm64v8"
;;
*)
container_arch="amd64"
;;
esac
container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/$container_arch/$INPUT_CONTAINER_NAME:$tag"
container_filename="$(echo $container_name | sed -e 's/\//-/g' -e 's/:/-/g').tar"
if [ -n "$INPUT_TARGET" ]; then
podman_options="$podman_options --target $INPUT_TARGET"
fi
if [ -n "$INPUT_DOCKERFILE" ]; then
podman_options="$podman_options -f $INPUT_DOCKERFILE"
fi
podman_options="$podman_options ${INPUT_CONTEXT:-.}"
echo "Podman Options: $podman_options"
podman build -t $container_name $podman_options
podman save $container_name > $container_filename
echo "container-full-name=$container_name" >> $GITHUB_OUTPUT
- name: Create container artifact
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
with:
name: ${{ inputs.container-name }}-${{ runner.arch }}
path: "*.tar"
retention-days: 14

- name: Test container
shell: bash
if: inputs.test-command
env:
INPUT_TEST_COMMAND: ${{ inputs.test-command }}
CONTAINER_FULL_NAME: ${{ steps.build.outputs.container-full-name }}
run: |
podman run --pull=never --rm -it $CONTAINER_FULL_NAME /usr/bin/bash -x -c "$INPUT_TEST_COMMAND"
Loading
Loading