diff --git a/.github/workflows/build-ci-container-tooling.yml b/.github/workflows/build-ci-container-tooling.yml index d3e33e2ccf931..46dc38fe600a3 100644 --- a/.github/workflows/build-ci-container-tooling.yml +++ b/.github/workflows/build-ci-container-tooling.yml @@ -12,17 +12,30 @@ on: - '.github/workflows/containers/github-action-ci-tooling/**' - llvm/utils/git/requirements_formatting.txt - llvm/utils/git/requirements_linting.txt + - '.github/workflows/build-container/**' + - '.github/workflows/push-container/**' pull_request: paths: - .github/workflows/build-ci-container-tooling.yml - '.github/workflows/containers/github-action-ci-tooling/**' - llvm/utils/git/requirements_formatting.txt - llvm/utils/git/requirements_linting.txt + - '.github/workflows/build-container/**' + - '.github/workflows/push-container/**' jobs: build-ci-container-tooling: + name: Build Container ${{ matrix.container-name }} if: github.repository_owner == 'llvm' runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - container-name: code-format + test-command: 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black' + - container-name: code-lint + test-command: 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage' steps: - name: Checkout LLVM uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -32,48 +45,15 @@ jobs: llvm/utils/git/requirements_formatting.txt llvm/utils/git/requirements_linting.txt clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py + .github/workflows/build-container - - name: Write Variables - id: vars - run: | - tag=$(git rev-parse --short=12 HEAD) - container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/amd64/ci-ubuntu-24.04" - echo "container-name-format=$container_name-code-format" >> $GITHUB_OUTPUT - echo "container-name-lint=$container_name-code-lint" >> $GITHUB_OUTPUT - echo "container-name-format-tag=$container_name-format:$tag" >> $GITHUB_OUTPUT - echo "container-name-lint-tag=$container_name-lint:$tag" >> $GITHUB_OUTPUT - echo "container-format-filename=$(echo $container_name-format:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT - echo "container-lint-filename=$(echo $container_name-lint:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT - - - name: Build container - run: | - podman build --target ci-container-code-format \ - -f .github/workflows/containers/github-action-ci-tooling/Dockerfile \ - -t ${{ steps.vars.outputs.container-name-format-tag }} . - podman build --target ci-container-code-lint \ - -f .github/workflows/containers/github-action-ci-tooling/Dockerfile \ - -t ${{ steps.vars.outputs.container-name-lint-tag }} . - - # Save the container so we have it in case the push fails. This also - # allows us to separate the push step into a different job so we can - # maintain minimal permissions while building the container. - - name: Save container image - run: | - podman save ${{ steps.vars.outputs.container-name-format-tag }} > ${{ steps.vars.outputs.container-format-filename }} - podman save ${{ steps.vars.outputs.container-name-lint-tag }} > ${{ steps.vars.outputs.container-lint-filename }} - - - name: Upload container image - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + - name: Build Container + uses: ./.github/workflows/build-container with: - name: container-amd64 - path: "*.tar" - retention-days: 14 - - - name: Test Container - run: | - # Use --pull=never to ensure we are testing the just built image. - podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-format-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black' - podman run --pull=never --rm -it ${{ steps.vars.outputs.container-name-lint-tag }} /usr/bin/bash -x -c 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage' + container-name: ci-ubuntu-24.04-${{ matrix.container-name }} + dockerfile: .github/workflows/containers/github-action-ci-tooling/Dockerfile + target: ci-container-${{ matrix.container-name }} + test-command: ${{ matrix.test-command }} push-ci-container: if: github.event_name == 'push' @@ -82,34 +62,13 @@ jobs: permissions: packages: write runs-on: ubuntu-24.04 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Download container - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - - - name: Push Container - run: | - function push_container { - image_name=$1 - latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g') - podman tag $image_name $latest_name - echo "Pushing $image_name ..." - podman push $image_name - echo "Pushing $latest_name ..." - podman push $latest_name - } - - podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io - for f in $(find . -iname '*.tar'); do - image_name=$(podman load -q -i $f | sed 's/Loaded image: //g') - push_container $image_name + - name: Checkout LLVM + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + sparse-checkout: | + .github/workflows/push-container - if echo $image_name | grep '/amd64/'; then - # For amd64, create an alias with the arch component removed. - # This matches the convention used on dockerhub. - default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name)) - podman tag $image_name $default_image_name - push_container $default_image_name - fi - done + - uses: ./.github/workflows/push-container + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml index 49c10d3318330..33b4dda2b2980 100644 --- a/.github/workflows/build-ci-container.yml +++ b/.github/workflows/build-ci-container.yml @@ -10,72 +10,46 @@ on: paths: - .github/workflows/build-ci-container.yml - '.github/workflows/containers/github-action-ci/**' + - '.github/workflows/build-container/**' + - '.github/workflows/push-container/**' pull_request: paths: - .github/workflows/build-ci-container.yml - '.github/workflows/containers/github-action-ci/**' + - '.github/workflows/build-container/**' + - '.github/workflows/push-container/**' jobs: build-ci-container: + name: Build Container ${{ matrix.container-name }} ${{ (contains(matrix.runs-on, 'arm') && 'ARM64') || 'X64' }} if: github.repository_owner == 'llvm' runs-on: ${{ matrix.runs-on }} strategy: matrix: - include: - # The arch names should match the names used on dockerhub. - # See https://github.com/docker-library/official-images#architectures-other-than-amd64 - - arch: amd64 - runs-on: depot-ubuntu-24.04-16 - - arch: arm64v8 - runs-on: depot-ubuntu-24.04-arm-16 + runs-on: + - depot-ubuntu-24.04-16 + - depot-ubuntu-24.04-arm-16 + container-name: + - '' + - agent + test-command: + - cd $HOME && printf '#include \nint main(int argc, char **argv) { std::cout << "Hello\\n"; }' | clang++ -x c++ - && ./a.out | grep Hello steps: - name: Checkout LLVM uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: - sparse-checkout: .github/workflows/containers/github-action-ci/ - # podman is not installed by default on the ARM64 images. - - name: Install Podman - if: runner.arch == 'ARM64' - run: | - sudo apt-get install podman - - name: Write Variables - id: vars - run: | - tag=$(git rev-parse --short=12 HEAD) - container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/${{ matrix.arch }}/ci-ubuntu-24.04" - echo "container-name=$container_name" >> $GITHUB_OUTPUT - echo "container-name-agent=$container_name-agent" >> $GITHUB_OUTPUT - echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT - echo "container-name-agent-tag=$container_name-agent:$tag" >> $GITHUB_OUTPUT - echo "container-filename=$(echo $container_name:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT - echo "container-agent-filename=$(echo $container_name-agent:$tag | sed -e 's/\//-/g' -e 's/:/-/g').tar" >> $GITHUB_OUTPUT - - name: Build container - working-directory: ./.github/workflows/containers/github-action-ci/ - run: | - podman build --target ci-container -t ${{ steps.vars.outputs.container-name-tag }} . - podman build --target ci-container-agent -t ${{ steps.vars.outputs.container-name-agent-tag }} . + sparse-checkout: | + .github/workflows/containers/github-action-ci/ + .github/workflows/build-container - # Save the container so we have it in case the push fails. This also - # allows us to separate the push step into a different job so we can - # maintain minimal permissions while building the container. - - name: Save container image - run: | - podman save ${{ steps.vars.outputs.container-name-tag }} > ${{ steps.vars.outputs.container-filename }} - podman save ${{ steps.vars.outputs.container-name-agent-tag }} > ${{ steps.vars.outputs.container-agent-filename }} - - - name: Upload container image - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + - name: Build Container + uses: ./.github/workflows/build-container with: - name: container-${{ matrix.arch }} - path: "*.tar" - retention-days: 14 - - - name: Test Container - run: | - for image in ${{ steps.vars.outputs.container-name-tag }}; do - # Use --pull=never to ensure we are testing the just built image. - podman run --pull=never --rm -it $image /usr/bin/bash -x -c 'cd $HOME && printf '\''#include \nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello' - done + container-name: ci-ubuntu-24.04${{ matrix.container-name && format('-{0}', matrix.container-name)}} + context: .github/workflows/containers/github-action-ci/ + dockerfile: .github/workflows/containers/github-action-ci/Dockerfile + target: ci-container${{ matrix.container-name && format('-{0}', matrix.container-name) }} + test-command: ${{ matrix.test-command }} push-ci-container: if: github.event_name == 'push' @@ -87,31 +61,12 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Download container - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - - - name: Push Container - run: | - function push_container { - image_name=$1 - latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g') - podman tag $image_name $latest_name - echo "Pushing $image_name ..." - podman push $image_name - echo "Pushing $latest_name ..." - podman push $latest_name - } - - podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io - for f in $(find . -iname '*.tar'); do - image_name=$(podman load -q -i $f | sed 's/Loaded image: //g') - push_container $image_name + - name: Checkout LLVM + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + sparse-checkout: | + .github/workflows/push-container - if echo $image_name | grep '/amd64/'; then - # For amd64, create an alias with the arch component removed. - # This matches the convention used on dockerhub. - default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name)) - podman tag $image_name $default_image_name - push_container $default_image_name - fi - done + - uses: ./.github/workflows/push-container + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-container/action.yml b/.github/workflows/build-container/action.yml new file mode 100644 index 0000000000000..595c3f8dd2070 --- /dev/null +++ b/.github/workflows/build-container/action.yml @@ -0,0 +1,95 @@ +name: Build Container +description: >- + Build and test a container using the standard llvm naming scheme for containers. + +inputs: + tag: + description: >- + The tag to use for this container. + required: false + container-name: + description: >- + The name for the container. + required: true + dockerfile: + description: >- + Path to docker file. + required: false + target: + description: >- + The container target to build 'passed to podman via ---target option' + required: false + context: + description: >- + Path to context for the container build. + required: false + test-command: + description: >- + Test command to run to ensure the container is working correctly. + required: false + +runs: + using: "composite" + steps: + # podman is not installed by default on the ARM64 images. + - name: Install Podman + if: runner.arch == 'ARM64' + shell: bash + run: | + sudo apt-get install podman + + - name: Build Container + shell: bash + env: + INPUT_TAG: ${{inputs.tag }} + INPUT_CONTAINER_NAME: ${{ inputs.container-name }} + INPUT_TARGET: ${{ inputs.target }} + INPUT_DOCKERFILE: ${{ inputs.dockerfile }} + INPUT_CONTEXT: ${{ inputs.context }} + id: build + run: | + env + tag="${INPUT_TAG:-$(git rev-parse --short=12 HEAD)}" + + case "$RUNNER_ARCH" in + ARM64) + container_arch="arm64v8" + ;; + *) + container_arch="amd64" + ;; + esac + + container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/$container_arch/$INPUT_CONTAINER_NAME:$tag" + container_filename="$(echo $container_name | sed -e 's/\//-/g' -e 's/:/-/g').tar" + if [ -n "$INPUT_TARGET" ]; then + podman_options="$podman_options --target $INPUT_TARGET" + fi + if [ -n "$INPUT_DOCKERFILE" ]; then + podman_options="$podman_options -f $INPUT_DOCKERFILE" + fi + podman_options="$podman_options ${INPUT_CONTEXT:-.}" + echo "Podman Options: $podman_options" + + podman build -t $container_name $podman_options + + podman save $container_name > $container_filename + + echo "container-full-name=$container_name" >> $GITHUB_OUTPUT + + - name: Create container artifact + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: ${{ inputs.container-name }}-${{ runner.arch }} + path: "*.tar" + retention-days: 14 + + - name: Test container + shell: bash + if: inputs.test-command + env: + INPUT_TEST_COMMAND: ${{ inputs.test-command }} + CONTAINER_FULL_NAME: ${{ steps.build.outputs.container-full-name }} + run: | + podman run --pull=never --rm -it $CONTAINER_FULL_NAME /usr/bin/bash -x -c "$INPUT_TEST_COMMAND" + diff --git a/.github/workflows/push-container/action.yml b/.github/workflows/push-container/action.yml new file mode 100644 index 0000000000000..51f4b2aa0002e --- /dev/null +++ b/.github/workflows/push-container/action.yml @@ -0,0 +1,44 @@ +name: Push Container +description: >- + Download all container artifacts for this job and push them to the GitHub registry. + +inputs: + token: + description: >- + Token to use to authenticate with the container registry. + required: true + +runs: + using: "composite" + steps: + - name: Download container + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + + - name: Push Container + env: + GITHUB_TOKEN: ${{ inputs.token }} + shell: bash + run: | + function push_container { + image_name=$1 + latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g') + podman tag $image_name $latest_name + echo "Pushing $image_name ..." + podman push $image_name + echo "Pushing $latest_name ..." + podman push $latest_name + } + + podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io + for f in $(find . -iname '*.tar'); do + image_name=$(podman load -q -i $f | sed 's/Loaded image: //g') + push_container $image_name + + if echo $image_name | grep '/amd64/'; then + # For amd64, create an alias with the arch component removed. + # This matches the convention used on dockerhub. + default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name)) + podman tag $image_name $default_image_name + push_container $default_image_name + fi + done