From a615f3bd08c9f0f32b679b561d52b24b89a897c0 Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 20 Nov 2025 20:06:32 -0800 Subject: [PATCH 01/14] Some updates to attempt to optimize our Dockerfile Successfully built locally, now to see if CI passes --- tools/ci-build/Dockerfile | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index d867cc9fe0..177ac51b0c 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -120,7 +120,8 @@ RUN cargo install --locked --path tools/ci-build/changelogger && \ cargo install --locked --path tools/ci-build/sdk-versioner && \ chmod g+rw -R /opt/cargo/registry && \ rm -rf /opt/cargo/registry/src && \ - rm -rf /opt/cargo/git/db + rm -rf /opt/cargo/git/db && \ + rm -rf /root/.cargo/registry/cache FROM install_rust AS cargo_tools ARG cargo_deny_version=0.16.4 @@ -149,7 +150,8 @@ RUN cargo install cargo-deny --locked --version ${cargo_deny_version} && \ cargo install mdbook --locked --version ${cargo_mdbook_version} && \ cargo install mdbook-mermaid --locked --version ${cargo_mdbook_mermaid_version} && \ rm -rf /opt/cargo/registry/src && \ - rm -rf /opt/cargo/git/db + rm -rf /opt/cargo/git/db && \ + rm -rf /root/.cargo/registry/cache # nodejs needed by internal release process FROM install_rust AS nodejs @@ -193,23 +195,19 @@ ARG rust_stable_version ARG rust_nightly_version RUN set -eux; \ yum -y install --allowerasing \ - bc \ ca-certificates \ clang \ gcc \ git \ glibc-langpack-en \ - go \ java-17-amazon-corretto-devel \ jq \ make \ openssl-devel \ pkgconfig \ python3 \ - python3-devel \ python3-pip \ shadow-utils \ - cmake \ tar \ unzip && \ yum clean all && \ @@ -224,21 +222,10 @@ RUN set -eux; \ rm -rf awscliv2.zip aws && \ pip3 install --no-cache-dir mypy==0.991 -COPY --chown=build:build --from=local_tools /opt/cargo /opt/cargo +COPY --chown=build:build --from=local_tools /opt/cargo/bin/ /opt/cargo/bin/ COPY --chown=build:build --from=gradle_wrapper /opt/gradle /home/build/.gradle -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/cargo-deny /opt/cargo/bin/cargo-deny -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/cargo-udeps /opt/cargo/bin/cargo-udeps -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/cargo-hack /opt/cargo/bin/cargo-hack -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/cargo-minimal-versions /opt/cargo/bin/cargo-minimal-versions -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/cargo-check-external-types /opt/cargo/bin/cargo-check-external-types -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/maturin /opt/cargo/bin/maturin -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/wasm-pack /opt/cargo/bin/wasm-pack -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/wasmtime /opt/cargo/bin/wasmtime -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/cargo-component /opt/cargo/bin/cargo-component +COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/ /opt/cargo/bin/ COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/cargo-semver-checks /opt/cargo/bin/cargo-semver-checks -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/mdbook /opt/cargo/bin/mdbook -COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/mdbook-mermaid /opt/cargo/bin/mdbook-mermaid COPY --chown=build:build --from=nodejs /opt/nodejs /opt/nodejs COPY --chown=build:build --from=musl_toolchain /usr/local/musl/ /usr/local/musl/ ENV PATH=/opt/nodejs/bin:/opt/cargo/bin:$PATH:/usr/local/musl/bin/ \ From 24c091acd6fb0ef3f155dead568859cb508cf97c Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 20 Nov 2025 21:59:37 -0800 Subject: [PATCH 02/14] Small update to not delete a cargo related folder --- tools/ci-build/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 177ac51b0c..ded98533d1 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -120,8 +120,7 @@ RUN cargo install --locked --path tools/ci-build/changelogger && \ cargo install --locked --path tools/ci-build/sdk-versioner && \ chmod g+rw -R /opt/cargo/registry && \ rm -rf /opt/cargo/registry/src && \ - rm -rf /opt/cargo/git/db && \ - rm -rf /root/.cargo/registry/cache + rm -rf /opt/cargo/git/db FROM install_rust AS cargo_tools ARG cargo_deny_version=0.16.4 @@ -150,8 +149,7 @@ RUN cargo install cargo-deny --locked --version ${cargo_deny_version} && \ cargo install mdbook --locked --version ${cargo_mdbook_version} && \ cargo install mdbook-mermaid --locked --version ${cargo_mdbook_mermaid_version} && \ rm -rf /opt/cargo/registry/src && \ - rm -rf /opt/cargo/git/db && \ - rm -rf /root/.cargo/registry/cache + rm -rf /opt/cargo/git/db # nodejs needed by internal release process FROM install_rust AS nodejs From cbda5b5cd0aa84d3909c069da970039046ab946a Mon Sep 17 00:00:00 2001 From: Landon James Date: Fri, 21 Nov 2025 11:21:40 -0800 Subject: [PATCH 03/14] Update local_tools copy --- tools/ci-build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index ded98533d1..f3754f24a7 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -220,7 +220,7 @@ RUN set -eux; \ rm -rf awscliv2.zip aws && \ pip3 install --no-cache-dir mypy==0.991 -COPY --chown=build:build --from=local_tools /opt/cargo/bin/ /opt/cargo/bin/ +COPY --chown=build:build --from=local_tools /opt/cargo /opt/cargo COPY --chown=build:build --from=gradle_wrapper /opt/gradle /home/build/.gradle COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/ /opt/cargo/bin/ COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup From d279a765b06454395310093fad0d57aeee40d7a8 Mon Sep 17 00:00:00 2001 From: Landon James Date: Fri, 21 Nov 2025 12:52:19 -0800 Subject: [PATCH 04/14] Update Dockerfile, add back go/py 3.9 --- tools/ci-build/Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index f3754f24a7..0e31db16be 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -120,7 +120,8 @@ RUN cargo install --locked --path tools/ci-build/changelogger && \ cargo install --locked --path tools/ci-build/sdk-versioner && \ chmod g+rw -R /opt/cargo/registry && \ rm -rf /opt/cargo/registry/src && \ - rm -rf /opt/cargo/git/db + rm -rf /opt/cargo/git/db && \ + rm -rf /root/.cargo/registry/cache FROM install_rust AS cargo_tools ARG cargo_deny_version=0.16.4 @@ -149,7 +150,8 @@ RUN cargo install cargo-deny --locked --version ${cargo_deny_version} && \ cargo install mdbook --locked --version ${cargo_mdbook_version} && \ cargo install mdbook-mermaid --locked --version ${cargo_mdbook_mermaid_version} && \ rm -rf /opt/cargo/registry/src && \ - rm -rf /opt/cargo/git/db + rm -rf /opt/cargo/git/db && \ + rm -rf /root/.cargo/registry/cache # nodejs needed by internal release process FROM install_rust AS nodejs @@ -198,12 +200,14 @@ RUN set -eux; \ gcc \ git \ glibc-langpack-en \ + go \ java-17-amazon-corretto-devel \ jq \ make \ openssl-devel \ pkgconfig \ python3 \ + python3-devel \ python3-pip \ shadow-utils \ tar \ From f557077ee4d499102b370f77ed51fc558686eb59 Mon Sep 17 00:00:00 2001 From: Landon James Date: Fri, 21 Nov 2025 14:43:12 -0800 Subject: [PATCH 05/14] Add back cmake --- tools/ci-build/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 0e31db16be..8afe258b3a 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -197,6 +197,7 @@ RUN set -eux; \ yum -y install --allowerasing \ ca-certificates \ clang \ + cmake \ gcc \ git \ glibc-langpack-en \ From 9bdea6d6f8392d6f412fee374986c842699b8a36 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 22 Nov 2025 20:55:22 -0800 Subject: [PATCH 06/14] Add bc back to image --- tools/ci-build/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 8afe258b3a..86c07e87b8 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -51,6 +51,7 @@ RUN set -eux; \ WORKDIR /root RUN yum -y install --allowerasing \ + bc \ autoconf \ automake \ binutils \ From 90834b80bdbae842582cab71dc471157cd8a522e Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 22 Nov 2025 20:57:53 -0800 Subject: [PATCH 07/14] Add .dockerignore --- tools/ci-build/.dockerignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tools/ci-build/.dockerignore diff --git a/tools/ci-build/.dockerignore b/tools/ci-build/.dockerignore new file mode 100644 index 0000000000..835de9f041 --- /dev/null +++ b/tools/ci-build/.dockerignore @@ -0,0 +1,5 @@ +target/ +*.md +.github/ +docs/ +examples/ From 77a545691096f2e3ba0d572e9356b2369a9303fd Mon Sep 17 00:00:00 2001 From: Landon James Date: Sun, 23 Nov 2025 17:14:10 -0800 Subject: [PATCH 08/14] Update .dockerignore --- tools/ci-build/.dockerignore | 2 +- tools/ci-build/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci-build/.dockerignore b/tools/ci-build/.dockerignore index 835de9f041..92d253085d 100644 --- a/tools/ci-build/.dockerignore +++ b/tools/ci-build/.dockerignore @@ -1,5 +1,5 @@ target/ *.md .github/ -docs/ +design/ examples/ diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 86c07e87b8..5dcb1594e2 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -51,7 +51,6 @@ RUN set -eux; \ WORKDIR /root RUN yum -y install --allowerasing \ - bc \ autoconf \ automake \ binutils \ @@ -196,6 +195,7 @@ ARG rust_stable_version ARG rust_nightly_version RUN set -eux; \ yum -y install --allowerasing \ + bc \ ca-certificates \ clang \ cmake \ From 173c7e1621aef92a00db8f40f732bb6537a30b32 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 24 Nov 2025 10:18:22 -0800 Subject: [PATCH 09/14] Update local_tools copy to only copy bin --- tools/ci-build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 5dcb1594e2..e50ab0718b 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -226,7 +226,7 @@ RUN set -eux; \ rm -rf awscliv2.zip aws && \ pip3 install --no-cache-dir mypy==0.991 -COPY --chown=build:build --from=local_tools /opt/cargo /opt/cargo +COPY --chown=build:build --from=local_tools /opt/cargo/bin /opt/cargo/bin COPY --chown=build:build --from=gradle_wrapper /opt/gradle /home/build/.gradle COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/ /opt/cargo/bin/ COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup From c6ebc0efb2e243c0d770a02e61dc0aa548cbfc5f Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 24 Nov 2025 10:59:26 -0800 Subject: [PATCH 10/14] Chown opt/cargo separate from COPY --- tools/ci-build/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index e50ab0718b..a4a7748fd7 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -224,7 +224,8 @@ RUN set -eux; \ unzip awscliv2.zip && \ ./aws/install && \ rm -rf awscliv2.zip aws && \ - pip3 install --no-cache-dir mypy==0.991 + pip3 install --no-cache-dir mypy==0.991 && \ + mkdir -p /opt/cargo && chown -R build:build /opt/cargo COPY --chown=build:build --from=local_tools /opt/cargo/bin /opt/cargo/bin COPY --chown=build:build --from=gradle_wrapper /opt/gradle /home/build/.gradle From d10136c98c0d89863dcd0704c606cb9b3c40a5fc Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 24 Nov 2025 11:29:15 -0800 Subject: [PATCH 11/14] Add small set of unused dep removals to CI workflows --- .github/actions/free-disk-space/action.yml | 13 +++++++++++++ .github/workflows/ci-main.yml | 1 + .github/workflows/ci-pr-forks.yml | 1 + .github/workflows/ci-pr.yml | 1 + 4 files changed, 16 insertions(+) create mode 100644 .github/actions/free-disk-space/action.yml diff --git a/.github/actions/free-disk-space/action.yml b/.github/actions/free-disk-space/action.yml new file mode 100644 index 0000000000..865ebfeb2c --- /dev/null +++ b/.github/actions/free-disk-space/action.yml @@ -0,0 +1,13 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: 'Free Disk Space' +description: 'Free up disk space on GitHub Actions runners' +runs: + using: 'composite' + steps: + - name: Free up disk space + shell: bash + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 4a9d9ed25c..3bd16dd60c 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -36,6 +36,7 @@ jobs: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 - name: Checkout uses: actions/checkout@v4 + - uses: ./.github/actions/free-disk-space - name: Acquire credentials uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/ci-pr-forks.yml b/.github/workflows/ci-pr-forks.yml index 339c957daa..056c2a238b 100644 --- a/.github/workflows/ci-pr-forks.yml +++ b/.github/workflows/ci-pr-forks.yml @@ -34,6 +34,7 @@ jobs: - uses: actions/checkout@v4 with: path: smithy-rs + - uses: ./smithy-rs/.github/actions/free-disk-space - name: Acquire base image id: acquire env: diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 931cda09e0..25b9b546d4 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -69,6 +69,7 @@ jobs: - uses: actions/checkout@v4 with: path: smithy-rs + - uses: ./smithy-rs/.github/actions/free-disk-space - name: Acquire base image id: acquire env: From 65e5adfc573488d154988d85af8d5c551321b08f Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 24 Nov 2025 13:14:08 -0800 Subject: [PATCH 12/14] setfacl for opt/cargo --- tools/ci-build/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index a4a7748fd7..1396e18b53 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -225,7 +225,8 @@ RUN set -eux; \ ./aws/install && \ rm -rf awscliv2.zip aws && \ pip3 install --no-cache-dir mypy==0.991 && \ - mkdir -p /opt/cargo && chown -R build:build /opt/cargo + mkdir -p /opt/cargo && chown -R build:build /opt/cargo && \ + setfacl -d -m u:build:rwx /opt/cargo && setfacl -d -m g:build:rwx /opt/cargo COPY --chown=build:build --from=local_tools /opt/cargo/bin /opt/cargo/bin COPY --chown=build:build --from=gradle_wrapper /opt/gradle /home/build/.gradle From 8c78745ea1703fd67ca08e438a61320082666c63 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 24 Nov 2025 13:29:58 -0800 Subject: [PATCH 13/14] install acl to get setfacl --- tools/ci-build/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 1396e18b53..c723b84145 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -195,6 +195,7 @@ ARG rust_stable_version ARG rust_nightly_version RUN set -eux; \ yum -y install --allowerasing \ + acl \ bc \ ca-certificates \ clang \ From d5252dbe7c3ddcb86c0a59849acb0ed13efdb9ea Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 24 Nov 2025 14:08:51 -0800 Subject: [PATCH 14/14] backing out the opt/cargo/bin change --- tools/ci-build/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index c723b84145..5dcb1594e2 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -195,7 +195,6 @@ ARG rust_stable_version ARG rust_nightly_version RUN set -eux; \ yum -y install --allowerasing \ - acl \ bc \ ca-certificates \ clang \ @@ -225,11 +224,9 @@ RUN set -eux; \ unzip awscliv2.zip && \ ./aws/install && \ rm -rf awscliv2.zip aws && \ - pip3 install --no-cache-dir mypy==0.991 && \ - mkdir -p /opt/cargo && chown -R build:build /opt/cargo && \ - setfacl -d -m u:build:rwx /opt/cargo && setfacl -d -m g:build:rwx /opt/cargo + pip3 install --no-cache-dir mypy==0.991 -COPY --chown=build:build --from=local_tools /opt/cargo/bin /opt/cargo/bin +COPY --chown=build:build --from=local_tools /opt/cargo /opt/cargo COPY --chown=build:build --from=gradle_wrapper /opt/gradle /home/build/.gradle COPY --chown=build:build --from=cargo_tools /opt/cargo/bin/ /opt/cargo/bin/ COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup