Skip to content

Commit 278abf0

Browse files
brettlangdonbrettlangdon
authored
ci: enable sccache in GHA build (#15234)
## Description Enables the use and caching of `sccache` to help improve build times. Seeing some of the linux/macos build times cut in half (4 -> 2 mins). ## Testing <!-- Describe your testing strategy or note what tests are included --> ## Risks <!-- Note any risks associated with this change, or "None" if no risks --> ## Additional Notes <!-- Any other information that would be helpful for reviewers -->
1 parent f042d4f commit 278abf0

File tree

2 files changed

+99
-41
lines changed

2 files changed

+99
-41
lines changed

‎.github/workflows/build_deploy.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,17 @@ jobs:
111111
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
112112
with:
113113
persist-credentials: false
114+
115+
- name: Enable sccache
116+
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # 0.0.9
117+
118+
- name: Add addtional GHA cache-related env vars
119+
uses: actions/github-script@v7
120+
with:
121+
script: |
122+
core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'])
123+
core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
124+
114125
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
115126
with:
116127
name: source-dist
@@ -128,6 +139,10 @@ jobs:
128139
- name: Install source package
129140
env:
130141
CMAKE_BUILD_PARALLEL_LEVEL: 12
142+
CARGO_BUILD_JOBS: 12
143+
SCCACHE_GHA_ENABLED: true
144+
SCCACHE_CACHE_SIZE: 1G
145+
DD_USE_SCCACHE: 1
131146
run: pip install dist/*.tar.gz
132147

133148
- name: Test the source package

‎.github/workflows/build_python_3.yml‎

Lines changed: 84 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -83,52 +83,26 @@ jobs:
8383
fail-fast: false
8484
matrix:
8585
include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}
86-
env:
87-
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
88-
CIBW_SKIP: ${{ inputs.cibw_skip }}
89-
CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
90-
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
91-
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
92-
CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest
93-
CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }}
94-
CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
95-
CIBW_BEFORE_ALL_LINUX: |
96-
if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
97-
curl -sSf https://sh.rustup.rs | sh -s -- -y;
98-
fi
99-
CIBW_ENVIRONMENT_LINUX: PATH=$HOME/.cargo/bin:$PATH CMAKE_BUILD_PARALLEL_LEVEL=24 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
100-
# SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
101-
# `platform.mac_ver()` reports incorrect MacOS version at 11.0
102-
# See: https://stackoverflow.com/a/65402241
103-
CIBW_ENVIRONMENT_MACOS: CMAKE_BUILD_PARALLEL_LEVEL=24 SYSTEM_VERSION_COMPAT=0 CMAKE_ARGS="-DNATIVE_TESTING=OFF" SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
104-
CIBW_ENVIRONMENT_WINDOWS: SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
105-
# cibuildwheel repair will copy anything's under /output directory from the
106-
# build container to the host machine. This is a bit hacky way, but seems
107-
# to be the only way getting debug symbols out from the container while
108-
# we don't mess up with RECORD file.
109-
CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
110-
mkdir -p /output/debugwheelhouse &&
111-
python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
112-
python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
113-
mkdir ./tempwheelhouse &&
114-
unzip -l {wheel} | grep '\.so' &&
115-
auditwheel repair -w ./tempwheelhouse {wheel} &&
116-
mv ./tempwheelhouse/*.whl {dest_dir} &&
117-
rm -rf ./tempwheelhouse
118-
CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
119-
mkdir -p ./debugwheelhouse &&
120-
python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
121-
python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
122-
MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
123-
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
124-
CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
125-
12686
steps:
12787
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
12888
with:
12989
persist-credentials: false
13090
fetch-depth: 0
13191

92+
- name: Enable sccache
93+
if: runner.os != 'Windows'
94+
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # 0.0.9
95+
with:
96+
disable_annotations: true
97+
98+
- name: Add addtional GHA cache-related env vars
99+
uses: actions/github-script@v7
100+
if: runner.os != 'Windows'
101+
with:
102+
script: |
103+
core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'])
104+
core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'])
105+
132106
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
133107
name: Install Python
134108
with:
@@ -144,7 +118,76 @@ jobs:
144118
uses: pypa/cibuildwheel@c923d83ad9c1bc00211c5041d0c3f73294ff88f6 # v3.1.4
145119
with:
146120
only: ${{ matrix.only }}
147-
121+
env:
122+
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE: ${{ needs.compute_version.outputs.library_version }}
123+
CIBW_SKIP: ${{ inputs.cibw_skip }}
124+
CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }}
125+
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
126+
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
127+
CIBW_MUSLLINUX_I686_IMAGE: ghcr.io/datadog/dd-trace-py/pypa_musllinux_1_2_i686:latest
128+
CIBW_BEFORE_ALL_WINDOWS: ${{ matrix.os == 'windows-latest' && 'rustup target add i686-pc-windows-msvc' || (matrix.os == 'windows-11-arm' && 'rustup target add aarch64-pc-windows-msvc') }}
129+
CIBW_BEFORE_ALL_MACOS: rustup target add aarch64-apple-darwin
130+
CIBW_BEFORE_ALL_LINUX: |
131+
if [[ "$(uname -m)-$(uname -i)-$(uname -o | tr '[:upper:]' '[:lower:]')-$(ldd --version 2>&1 | head -n 1 | awk '{print $1}')" != "i686-unknown-linux-musl" ]]; then
132+
curl -sSf https://sh.rustup.rs | sh -s -- -y;
133+
fi
134+
CIBW_ENVIRONMENT_LINUX: >
135+
PATH=$HOME/.cargo/bin:$PATH
136+
CARGO_BUILD_JOBS=24
137+
CMAKE_BUILD_PARALLEL_LEVEL=24
138+
CMAKE_ARGS="-DNATIVE_TESTING=OFF"
139+
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
140+
SCCACHE_GHA_ENABLED=true
141+
SCCACHE_DIR=/host/${{ env.SCCACHE_DIR }}
142+
SCCACHE_PATH=/host/${{ env.SCCACHE_PATH }}
143+
SCCACHE_CACHE_SIZE=1G
144+
ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
145+
ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
146+
ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
147+
ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
148+
ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
149+
DD_USE_SCCACHE=1
150+
# SYSTEM_VERSION_COMPAT is a workaround for versioning issue, a.k.a.
151+
# `platform.mac_ver()` reports incorrect MacOS version at 11.0
152+
# See: https://stackoverflow.com/a/65402241
153+
CIBW_ENVIRONMENT_MACOS: >
154+
CMAKE_BUILD_PARALLEL_LEVEL=24
155+
CARGO_BUILD_JOBS=24
156+
SYSTEM_VERSION_COMPAT=0
157+
CMAKE_ARGS="-DNATIVE_TESTING=OFF"
158+
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
159+
SCCACHE_GHA_ENABLED=true
160+
SCCACHE_CACHE_SIZE=1G
161+
ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
162+
ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
163+
ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
164+
ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
165+
ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
166+
DD_USE_SCCACHE=1
167+
CIBW_ENVIRONMENT_WINDOWS: >
168+
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DDTRACE=${{ needs.compute_version.outputs.library_version }}
169+
# cibuildwheel repair will copy anything's under /output directory from the
170+
# build container to the host machine. This is a bit hacky way, but seems
171+
# to be the only way getting debug symbols out from the container while
172+
# we don't mess up with RECORD file.
173+
CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
174+
mkdir -p /output/debugwheelhouse &&
175+
python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
176+
python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
177+
mkdir ./tempwheelhouse &&
178+
unzip -l {wheel} | grep '\.so' &&
179+
auditwheel repair -w ./tempwheelhouse {wheel} &&
180+
mv ./tempwheelhouse/*.whl {dest_dir} &&
181+
rm -rf ./tempwheelhouse
182+
CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
183+
mkdir -p ./debugwheelhouse &&
184+
python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
185+
python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
186+
MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
187+
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
188+
CIBW_BEFORE_TEST_LINUX: /host/${{ env.SCCACHE_PATH }} --show-stats
189+
CIBW_BEFORE_TEST_MACOS: ${{ env.SCCACHE_PATH }} --show-stats
190+
CIBW_TEST_COMMAND: "python {project}/tests/smoke_test.py"
148191
- name: Validate wheel RECORD files
149192
shell: bash
150193
run: |

0 commit comments

Comments
 (0)